02:05:55.120 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
02:05:55.137 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskONRpAo/test-tmp
02:05:55.141 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
02:05:55.145 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskONRpAo/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
02:05:55.437 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
02:05:55.516 [DEBUG - main] (MiniKuduCluster.java:195) Response:
02:05:55.517 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
02:05:55.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:55.520617 32589 mini_oidc.cc:150] Starting JWKS server
02:05:55.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:55.524516 32589 webserver.cc:480] Webserver started at http://127.0.0.1:33831/ using document root <none> and password file <none>
02:05:55.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:55.525025 32589 mini_oidc.cc:186] Starting OIDC Discovery server
02:05:55.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:55.526806 32589 webserver.cc:480] Webserver started at http://127.0.0.1:42325/ using document root <none> and password file <none>
02:05:55.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-02T02:05:55Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
02:05:55.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-02T02:05:55Z Disabled control of system clock
02:05:55.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:55.587033 32589 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu
02:05:55.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu
02:05:55.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-0/wal
02:05:55.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-0/data
02:05:55.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:05:55.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:05:55.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:05:55.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:05:55.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:05:55.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:05:55.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-0/logs
02:05:55.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-0/data/info.pb
02:05:55.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:05:55.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:05:55.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:05:55.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:05:55.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:05:55.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:05:55.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:05:55.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:05:55.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:05:55.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:05:55.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:05:55.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:05:55.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.211.126:43055
02:05:55.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.211.126
02:05:55.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:05:55.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.211.84:35601
02:05:55.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:05:55.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:05:55.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:05:55.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:05:55.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.31.211.126:43055,127.31.211.125:43493,127.31.211.124:42871
02:05:55.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:05:55.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33831/jwks.json with env {}
02:05:55.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:55.880332 32601 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:05:55.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:55.880898 32601 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:05:55.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:55.881613 32601 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:05:55.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:55.914892 32601 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:05:55.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:55.915349 32601 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:05:55.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:55.915774 32601 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:05:55.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:55.916137 32601 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:05:55.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:55.916493 32601 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:05:55.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:55.916780 32601 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33831/jwks.json
02:05:55.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:55.979511 32601 master_runner.cc:386] Master server non-default flags:
02:05:55.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:05:55.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.211.84:35601
02:05:55.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:05:55.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:05:55.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-0/data
02:05:55.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-0/wal
02:05:55.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:05:55.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.31.211.126:43055,127.31.211.125:43493,127.31.211.124:42871
02:05:55.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:05:55.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:05:55.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:05:55.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.211.126:43055
02:05:55.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:05:55.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:05:55.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:05:55.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33831/jwks.json
02:05:55.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:05:55.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:05:55.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-0/data/info.pb
02:05:55.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.211.126
02:05:55.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:05:55.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:05:55.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:05:55.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:05:55.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:05:55.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:05:55.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-0/logs
02:05:55.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:05:55.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:05:55.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:05:55.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:05:55.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:05:55.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
02:05:55.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:05:55.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 01:43:19 UTC on 5fd53c4cbb9d
02:05:55.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7857
02:05:55.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:05:55.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:55.981245 32601 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:05:55.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:55.983485 32601 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:05:55.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:55.995432 32607 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:05:57.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:55.999121 32610 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:05:57.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:55.997392 32608 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:05:57.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:57.189740 32609 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1188 milliseconds
02:05:57.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:57.189862 32601 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:05:57.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:57.191251 32601 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:05:57.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:57.194458 32601 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:05:57.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:57.195937 32601 hybrid_clock.cc:648] HybridClock initialized: now 1756778757195890 us; error 60 us; skew 500 ppm
02:05:57.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:57.196864 32601 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:05:57.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:57.203040 32601 webserver.cc:480] Webserver started at http://127.31.211.126:39133/ using document root <none> and password file <none>
02:05:57.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:57.203977 32601 fs_manager.cc:362] Metadata directory not provided
02:05:57.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:57.204172 32601 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:05:57.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:57.204722 32601 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
02:05:57.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:57.209162 32601 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-0/data/instance:
02:05:57.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5965108c7d9b4918aff8b619718a23d7"
02:05:57.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 02:05:57 on dist-test-slave-szd5"
02:05:57.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:57.210327 32601 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-0/wal/instance:
02:05:57.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5965108c7d9b4918aff8b619718a23d7"
02:05:57.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 02:05:57 on dist-test-slave-szd5"
02:05:57.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:57.217547 32601 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
02:05:57.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:57.222949 32617 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:05:57.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:57.224085 32601 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
02:05:57.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:57.224494 32601 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-0/data,/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-0/wal
02:05:57.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5965108c7d9b4918aff8b619718a23d7"
02:05:57.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 02:05:57 on dist-test-slave-szd5"
02:05:57.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:57.224915 32601 fs_report.cc:389] FS layout report
02:05:57.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:05:57.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-0/wal
02:05:57.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-0/wal
02:05:57.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-0/data/data
02:05:57.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:05:57.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:05:57.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:05:57.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:05:57.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:05:57.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:05:57.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:05:57.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:05:57.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:05:57.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:05:57.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:05:57.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:57.275239 32601 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:05:57.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:57.276927 32601 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:05:57.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:57.277441 32601 kserver.cc:163] Server-wide thread pool size limit: 3276
02:05:57.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:57.352079 32601 rpc_server.cc:307] RPC server started. Bound to: 127.31.211.126:43055
02:05:57.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:57.352149 32668 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.31.211.126:43055 every 8 connection(s)
02:05:57.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:57.354665 32601 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-0/data/info.pb
02:05:57.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:57.360448 32669 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:05:57.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:57.360767 32589 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu as pid 32601
02:05:57.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:57.361296 32589 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-0/wal/instance
02:05:57.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:57.366609 32589 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu
02:05:57.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu
02:05:57.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-1/wal
02:05:57.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-1/data
02:05:57.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:05:57.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:05:57.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:05:57.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:05:57.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:05:57.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:05:57.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-1/logs
02:05:57.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-1/data/info.pb
02:05:57.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:05:57.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:05:57.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:05:57.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:05:57.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:05:57.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:05:57.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:05:57.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:05:57.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:05:57.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:05:57.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:05:57.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:05:57.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.211.125:43493
02:05:57.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.211.125
02:05:57.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:05:57.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.211.84:35601
02:05:57.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:05:57.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:05:57.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:05:57.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:05:57.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.31.211.126:43055,127.31.211.125:43493,127.31.211.124:42871
02:05:57.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:05:57.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33831/jwks.json with env {}
02:05:57.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:57.376507 32669 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.211.126" port: 43055 } has no permanent_uuid. Determining permanent_uuid...
02:05:57.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:57.405349 32669 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.211.125" port: 43493 } has no permanent_uuid. Determining permanent_uuid...
02:05:57.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:57.408428 32620 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.31.211.125:43493: connect: Connection refused (error 111)
02:05:57.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:57.412916 32669 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.31.211.125:43493: Network error: Client connection negotiation failed: client connection to 127.31.211.125:43493: connect: Connection refused (error 111)
02:05:57.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:57.462512 32669 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.211.125" port: 43493 } attempt: 1
02:05:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:57.467017 32669 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.31.211.125:43493: Network error: Client connection negotiation failed: client connection to 127.31.211.125:43493: connect: Connection refused (error 111)
02:05:57.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:57.535449 32669 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.211.125" port: 43493 } attempt: 2
02:05:57.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:57.539916 32669 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.31.211.125:43493: Network error: Client connection negotiation failed: client connection to 127.31.211.125:43493: connect: Connection refused (error 111)
02:05:57.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:57.631297 32669 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.211.125" port: 43493 } attempt: 3
02:05:57.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:57.635341 32669 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.31.211.125:43493: Network error: Client connection negotiation failed: client connection to 127.31.211.125:43493: connect: Connection refused (error 111)
02:05:57.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:57.678968 32671 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:05:57.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:57.679523 32671 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:05:57.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:57.680140 32671 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:05:57.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:57.711122 32671 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:05:57.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:57.711411 32671 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:05:57.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:57.711673 32671 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:05:57.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:57.711872 32671 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:05:57.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:57.712046 32671 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:05:57.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:57.712178 32671 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33831/jwks.json
02:05:57.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:57.747483 32671 master_runner.cc:386] Master server non-default flags:
02:05:57.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:05:57.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.211.84:35601
02:05:57.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:05:57.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:05:57.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-1/data
02:05:57.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-1/wal
02:05:57.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:05:57.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.31.211.126:43055,127.31.211.125:43493,127.31.211.124:42871
02:05:57.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:05:57.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:05:57.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:05:57.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.211.125:43493
02:05:57.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:05:57.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:05:57.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:05:57.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33831/jwks.json
02:05:57.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:05:57.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:05:57.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-1/data/info.pb
02:05:57.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.211.125
02:05:57.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:05:57.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:05:57.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:05:57.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:05:57.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:05:57.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:05:57.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-1/logs
02:05:57.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:05:57.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:05:57.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:05:57.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:05:57.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:05:57.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
02:05:57.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:05:57.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 01:43:19 UTC on 5fd53c4cbb9d
02:05:57.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7857
02:05:57.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:05:57.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:57.748888 32671 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:05:57.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:57.751055 32671 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:05:57.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:57.762375 32680 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:05:57.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:57.778800 32669 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.211.125" port: 43493 } attempt: 4
02:05:57.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:57.787039 32669 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.31.211.125:43493: Network error: Client connection negotiation failed: client connection to 127.31.211.125:43493: connect: Connection refused (error 111)
02:05:58.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:58.086617 32669 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.211.125" port: 43493 } attempt: 5
02:05:58.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:58.094662 32669 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.31.211.125:43493: Network error: Client connection negotiation failed: client connection to 127.31.211.125:43493: connect: Connection refused (error 111)
02:05:58.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:58.642272 32669 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.211.125" port: 43493 } attempt: 6
02:05:58.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:58.669423 32669 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.31.211.125:43493: Network error: Client connection negotiation failed: client connection to 127.31.211.125:43493: connect: Connection refused (error 111)
02:05:59.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:59.167724 32679 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ec0 after lost signal to thread 32671
02:05:59.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:57.763062 32681 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:05:59.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:59.242142 32671 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.479s user 0.518s sys 0.956s
02:05:59.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:59.243780 32671 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.481s user 0.518s sys 0.956s
02:05:59.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:59.245049 32683 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:05:59.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:59.248060 32682 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1479 milliseconds
02:05:59.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:59.248109 32671 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:05:59.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:59.249351 32671 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:05:59.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:59.251776 32671 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:05:59.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:59.253121 32671 hybrid_clock.cc:648] HybridClock initialized: now 1756778759253076 us; error 43 us; skew 500 ppm
02:05:59.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:59.253955 32671 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:05:59.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:59.260020 32671 webserver.cc:480] Webserver started at http://127.31.211.125:38061/ using document root <none> and password file <none>
02:05:59.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:59.260962 32671 fs_manager.cc:362] Metadata directory not provided
02:05:59.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:59.261183 32671 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:05:59.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:59.261711 32671 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
02:05:59.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:59.266115 32671 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-1/data/instance:
02:05:59.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2bc5891afdc547f3ba433776f76f9f44"
02:05:59.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 02:05:59 on dist-test-slave-szd5"
02:05:59.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:59.267311 32671 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-1/wal/instance:
02:05:59.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2bc5891afdc547f3ba433776f76f9f44"
02:05:59.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 02:05:59 on dist-test-slave-szd5"
02:05:59.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:59.274258 32671 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.006s sys 0.000s
02:05:59.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:59.279626 32692 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:05:59.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:59.280725 32671 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
02:05:59.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:59.281065 32671 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-1/data,/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-1/wal
02:05:59.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2bc5891afdc547f3ba433776f76f9f44"
02:05:59.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 02:05:59 on dist-test-slave-szd5"
02:05:59.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:59.281448 32671 fs_report.cc:389] FS layout report
02:05:59.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:05:59.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-1/wal
02:05:59.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-1/wal
02:05:59.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-1/data/data
02:05:59.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:05:59.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:05:59.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:05:59.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:05:59.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:05:59.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:05:59.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:05:59.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:05:59.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:05:59.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:05:59.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:05:59.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:59.333061 32671 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:05:59.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:59.334630 32671 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:05:59.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:59.335139 32671 kserver.cc:163] Server-wide thread pool size limit: 3276
02:05:59.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:59.410367 32671 rpc_server.cc:307] RPC server started. Bound to: 127.31.211.125:43493
02:05:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:59.410406 32743 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.31.211.125:43493 every 8 connection(s)
02:05:59.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:59.413149 32671 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-1/data/info.pb
02:05:59.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:59.419130 32744 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:05:59.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:59.420888 32589 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu as pid 32671
02:05:59.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:59.421371 32589 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-1/wal/instance
02:05:59.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:59.424256 32589 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu
02:05:59.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu
02:05:59.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-2/wal
02:05:59.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-2/data
02:05:59.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:05:59.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:05:59.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:05:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:05:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:05:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:05:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-2/logs
02:05:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-2/data/info.pb
02:05:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:05:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:05:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:05:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:05:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:05:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:05:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:05:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:05:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:05:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:05:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:05:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:05:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.211.124:42871
02:05:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.211.124
02:05:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:05:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.211.84:35601
02:05:59.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:05:59.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:05:59.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:05:59.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:05:59.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.31.211.126:43055,127.31.211.125:43493,127.31.211.124:42871
02:05:59.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:05:59.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33831/jwks.json with env {}
02:05:59.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:59.433333 32744 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.211.126" port: 43055 } has no permanent_uuid. Determining permanent_uuid...
02:05:59.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:59.453749 32744 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.211.125" port: 43493 } has no permanent_uuid. Determining permanent_uuid...
02:05:59.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:59.467777 32744 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.211.124" port: 42871 } has no permanent_uuid. Determining permanent_uuid...
02:05:59.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:59.470295 32694 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.31.211.124:42871: connect: Connection refused (error 111)
02:05:59.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:59.474862 32744 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.31.211.124:42871: Network error: Client connection negotiation failed: client connection to 127.31.211.124:42871: connect: Connection refused (error 111)
02:05:59.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:59.524303 32744 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.211.124" port: 42871 } attempt: 1
02:05:59.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:59.528589 32744 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.31.211.124:42871: Network error: Client connection negotiation failed: client connection to 127.31.211.124:42871: connect: Connection refused (error 111)
02:05:59.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:59.596993 32744 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.211.124" port: 42871 } attempt: 2
02:05:59.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:59.601248 32744 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.31.211.124:42871: Network error: Client connection negotiation failed: client connection to 127.31.211.124:42871: connect: Connection refused (error 111)
02:05:59.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:59.692662 32744 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.211.124" port: 42871 } attempt: 3
02:05:59.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:59.696933 32744 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.31.211.124:42871: Network error: Client connection negotiation failed: client connection to 127.31.211.124:42871: connect: Connection refused (error 111)
02:05:59.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:59.728125 32746 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:05:59.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:59.728809 32746 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:05:59.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:59.729561 32746 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:05:59.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:59.730012 32669 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.211.125" port: 43493 } attempt: 7
02:05:59.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:59.742022 32669 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.211.124" port: 42871 } has no permanent_uuid. Determining permanent_uuid...
02:05:59.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:59.747741 32669 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.31.211.124:42871: Network error: Client connection negotiation failed: client connection to 127.31.211.124:42871: connect: Connection refused (error 111)
02:05:59.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:59.766559 32746 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:05:59.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:59.766841 32746 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:05:59.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:59.767066 32746 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:05:59.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:59.767346 32746 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:05:59.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:59.767697 32746 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:05:59.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:59.767946 32746 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33831/jwks.json
02:05:59.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:59.803743 32746 master_runner.cc:386] Master server non-default flags:
02:05:59.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:05:59.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.211.84:35601
02:05:59.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:05:59.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:05:59.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-2/data
02:05:59.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-2/wal
02:05:59.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:05:59.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.31.211.126:43055,127.31.211.125:43493,127.31.211.124:42871
02:05:59.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:05:59.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:05:59.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:05:59.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.211.124:42871
02:05:59.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:05:59.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:05:59.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:05:59.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33831/jwks.json
02:05:59.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:05:59.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:05:59.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-2/data/info.pb
02:05:59.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.211.124
02:05:59.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:05:59.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:05:59.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:05:59.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:05:59.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:05:59.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:05:59.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-2/logs
02:05:59.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:05:59.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:05:59.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:05:59.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:05:59.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:05:59.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
02:05:59.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:05:59.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 01:43:19 UTC on 5fd53c4cbb9d
02:05:59.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7857
02:05:59.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:05:59.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:59.805265 32746 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:05:59.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:59.806187 32669 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.211.124" port: 42871 } attempt: 1
02:05:59.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:59.807206 32746 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:05:59.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:59.811348 32669 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.31.211.124:42871: Network error: Client connection negotiation failed: client connection to 127.31.211.124:42871: connect: Connection refused (error 111)
02:05:59.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:59.819681 32758 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:05:59.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:59.840596 32744 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.211.124" port: 42871 } attempt: 4
02:05:59.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:59.847807 32744 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.31.211.124:42871: Network error: Client connection negotiation failed: client connection to 127.31.211.124:42871: connect: Connection refused (error 111)
02:05:59.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:59.892856 32669 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.211.124" port: 42871 } attempt: 2
02:05:59.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:59.902678 32669 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.31.211.124:42871: Network error: Client connection negotiation failed: client connection to 127.31.211.124:42871: connect: Connection refused (error 111)
02:05:59.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:05:59.988334 32669 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.211.124" port: 42871 } attempt: 3
02:05:59.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:59.996285 32669 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.31.211.124:42871: Network error: Client connection negotiation failed: client connection to 127.31.211.124:42871: connect: Connection refused (error 111)
02:06:00.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:00.136847 32669 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.211.124" port: 42871 } attempt: 4
02:06:00.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:00.144912 32669 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.31.211.124:42871: Network error: Client connection negotiation failed: client connection to 127.31.211.124:42871: connect: Connection refused (error 111)
02:06:00.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:00.147373 32744 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.211.124" port: 42871 } attempt: 5
02:06:00.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:00.155095 32744 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.31.211.124:42871: Network error: Client connection negotiation failed: client connection to 127.31.211.124:42871: connect: Connection refused (error 111)
02:06:00.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:00.428453 32669 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.211.124" port: 42871 } attempt: 5
02:06:00.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:00.435880 32669 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.31.211.124:42871: Network error: Client connection negotiation failed: client connection to 127.31.211.124:42871: connect: Connection refused (error 111)
02:06:00.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:00.702682 32744 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.211.124" port: 42871 } attempt: 6
02:06:00.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:00.733035 32744 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.31.211.124:42871: Network error: Client connection negotiation failed: client connection to 127.31.211.124:42871: connect: Connection refused (error 111)
02:06:00.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:00.988468 32669 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.211.124" port: 42871 } attempt: 6
02:06:01.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:01.011410 32669 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.31.211.124:42871: Network error: Client connection negotiation failed: client connection to 127.31.211.124:42871: connect: Connection refused (error 111)
02:06:01.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:01.224860 32757 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ec0 after lost signal to thread 32746
02:06:01.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:01.323269 32757 kernel_stack_watchdog.cc:198] Thread 32746 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 401ms:
02:06:01.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:06:01.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:06:01.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:06:01.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:06:01.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:01.324015 32746 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.504s user 0.538s sys 0.963s
02:06:01.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:01.324440 32746 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.505s user 0.538s sys 0.963s
02:06:01.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:05:59.820297 32759 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:01.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:01.325675 32760 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1505 milliseconds
02:06:01.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:01.326665 32761 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:01.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.326614 32746 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:06:01.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.330327 32746 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:06:01.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.333362 32746 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:06:01.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.335119 32746 hybrid_clock.cc:648] HybridClock initialized: now 1756778761335065 us; error 50 us; skew 500 ppm
02:06:01.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.336203 32746 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:06:01.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.343422 32746 webserver.cc:480] Webserver started at http://127.31.211.124:40867/ using document root <none> and password file <none>
02:06:01.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.344319 32746 fs_manager.cc:362] Metadata directory not provided
02:06:01.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.344525 32746 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:06:01.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.344893 32746 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
02:06:01.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.349062 32746 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-2/data/instance:
02:06:01.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "21e44f6f40f44af3b10c7af4fe478c17"
02:06:01.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 02:06:01 on dist-test-slave-szd5"
02:06:01.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.350348 32746 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-2/wal/instance:
02:06:01.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "21e44f6f40f44af3b10c7af4fe478c17"
02:06:01.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 02:06:01 on dist-test-slave-szd5"
02:06:01.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.360556 32746 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.008s sys 0.000s
02:06:01.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.368328 304 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:06:01.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.369467 32746 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.003s
02:06:01.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.369748 32746 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-2/data,/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-2/wal
02:06:01.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "21e44f6f40f44af3b10c7af4fe478c17"
02:06:01.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 02:06:01 on dist-test-slave-szd5"
02:06:01.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.370029 32746 fs_report.cc:389] FS layout report
02:06:01.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:06:01.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-2/wal
02:06:01.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-2/wal
02:06:01.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-2/data/data
02:06:01.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:06:01.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:06:01.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:06:01.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:06:01.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:06:01.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:06:01.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:06:01.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:06:01.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:06:01.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:06:01.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:01.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.443387 32746 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:06:01.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.444834 32746 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:01.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.445236 32746 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:01.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.520862 32746 rpc_server.cc:307] RPC server started. Bound to: 127.31.211.124:42871
02:06:01.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.520938 355 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.31.211.124:42871 every 8 connection(s)
02:06:01.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.523275 32746 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-2/data/info.pb
02:06:01.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.524436 32589 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu as pid 32746
02:06:01.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.524871 32589 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/master-2/wal/instance
02:06:01.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.530256 356 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:06:01.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.546272 356 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.211.126" port: 43055 } has no permanent_uuid. Determining permanent_uuid...
02:06:01.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.567188 356 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.211.125" port: 43493 } has no permanent_uuid. Determining permanent_uuid...
02:06:01.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.579558 356 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.211.124" port: 42871 } has no permanent_uuid. Determining permanent_uuid...
02:06:01.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.594736 356 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 21e44f6f40f44af3b10c7af4fe478c17: Bootstrap starting.
02:06:01.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.601203 356 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 21e44f6f40f44af3b10c7af4fe478c17: Neither blocks nor log segments found. Creating new log.
02:06:01.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.602854 356 log.cc:826] T 00000000000000000000000000000000 P 21e44f6f40f44af3b10c7af4fe478c17: Log is configured to *not* fsync() on all Append() calls
02:06:01.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.607128 356 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 21e44f6f40f44af3b10c7af4fe478c17: No bootstrap required, opened a new log
02:06:01.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.626089 356 raft_consensus.cc:357] T 00000000000000000000000000000000 P 21e44f6f40f44af3b10c7af4fe478c17 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5965108c7d9b4918aff8b619718a23d7" member_type: VOTER last_known_addr { host: "127.31.211.126" port: 43055 } } peers { permanent_uuid: "2bc5891afdc547f3ba433776f76f9f44" member_type: VOTER last_known_addr { host: "127.31.211.125" port: 43493 } } peers { permanent_uuid: "21e44f6f40f44af3b10c7af4fe478c17" member_type: VOTER last_known_addr { host: "127.31.211.124" port: 42871 } }
02:06:01.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.626699 356 raft_consensus.cc:383] T 00000000000000000000000000000000 P 21e44f6f40f44af3b10c7af4fe478c17 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:06:01.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.626883 356 raft_consensus.cc:738] T 00000000000000000000000000000000 P 21e44f6f40f44af3b10c7af4fe478c17 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 21e44f6f40f44af3b10c7af4fe478c17, State: Initialized, Role: FOLLOWER
02:06:01.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.627738 356 consensus_queue.cc:260] T 00000000000000000000000000000000 P 21e44f6f40f44af3b10c7af4fe478c17 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5965108c7d9b4918aff8b619718a23d7" member_type: VOTER last_known_addr { host: "127.31.211.126" port: 43055 } } peers { permanent_uuid: "2bc5891afdc547f3ba433776f76f9f44" member_type: VOTER last_known_addr { host: "127.31.211.125" port: 43493 } } peers { permanent_uuid: "21e44f6f40f44af3b10c7af4fe478c17" member_type: VOTER last_known_addr { host: "127.31.211.124" port: 42871 } }
02:06:01.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.630839 364 sys_catalog.cc:455] T 00000000000000000000000000000000 P 21e44f6f40f44af3b10c7af4fe478c17 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5965108c7d9b4918aff8b619718a23d7" member_type: VOTER last_known_addr { host: "127.31.211.126" port: 43055 } } peers { permanent_uuid: "2bc5891afdc547f3ba433776f76f9f44" member_type: VOTER last_known_addr { host: "127.31.211.125" port: 43493 } } peers { permanent_uuid: "21e44f6f40f44af3b10c7af4fe478c17" member_type: VOTER last_known_addr { host: "127.31.211.124" port: 42871 } } }
02:06:01.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.631625 364 sys_catalog.cc:458] T 00000000000000000000000000000000 P 21e44f6f40f44af3b10c7af4fe478c17 [sys.catalog]: This master's current role is: FOLLOWER
02:06:01.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.632694 356 sys_catalog.cc:564] T 00000000000000000000000000000000 P 21e44f6f40f44af3b10c7af4fe478c17 [sys.catalog]: configured and running, proceeding with master startup.
02:06:01.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:01.654650 375 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 21e44f6f40f44af3b10c7af4fe478c17: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:06:01.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:01.654924 375 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:06:01.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.651925 32746 master_runner.cc:186] Error getting master registration for 127.31.211.126:43055: OK, instance_id { permanent_uuid: "5965108c7d9b4918aff8b619718a23d7" instance_seqno: 1756778757333458 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:06:01.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.681298 32746 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:06:01.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:01.701182 310 tablet.cc:2378] T 00000000000000000000000000000000 P 21e44f6f40f44af3b10c7af4fe478c17: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:06:01.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.793586 32744 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.211.124" port: 42871 } attempt: 7
02:06:01.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.809674 32744 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2bc5891afdc547f3ba433776f76f9f44: Bootstrap starting.
02:06:01.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.816358 32744 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 2bc5891afdc547f3ba433776f76f9f44: Neither blocks nor log segments found. Creating new log.
02:06:01.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.818379 32744 log.cc:826] T 00000000000000000000000000000000 P 2bc5891afdc547f3ba433776f76f9f44: Log is configured to *not* fsync() on all Append() calls
02:06:01.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.819298 364 raft_consensus.cc:491] T 00000000000000000000000000000000 P 21e44f6f40f44af3b10c7af4fe478c17 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:06:01.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.819921 364 raft_consensus.cc:513] T 00000000000000000000000000000000 P 21e44f6f40f44af3b10c7af4fe478c17 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5965108c7d9b4918aff8b619718a23d7" member_type: VOTER last_known_addr { host: "127.31.211.126" port: 43055 } } peers { permanent_uuid: "2bc5891afdc547f3ba433776f76f9f44" member_type: VOTER last_known_addr { host: "127.31.211.125" port: 43493 } } peers { permanent_uuid: "21e44f6f40f44af3b10c7af4fe478c17" member_type: VOTER last_known_addr { host: "127.31.211.124" port: 42871 } }
02:06:01.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.822820 364 leader_election.cc:290] T 00000000000000000000000000000000 P 21e44f6f40f44af3b10c7af4fe478c17 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5965108c7d9b4918aff8b619718a23d7 (127.31.211.126:43055), 2bc5891afdc547f3ba433776f76f9f44 (127.31.211.125:43493)
02:06:01.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.823597 32744 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2bc5891afdc547f3ba433776f76f9f44: No bootstrap required, opened a new log
02:06:01.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.849493 32744 raft_consensus.cc:357] T 00000000000000000000000000000000 P 2bc5891afdc547f3ba433776f76f9f44 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5965108c7d9b4918aff8b619718a23d7" member_type: VOTER last_known_addr { host: "127.31.211.126" port: 43055 } } peers { permanent_uuid: "2bc5891afdc547f3ba433776f76f9f44" member_type: VOTER last_known_addr { host: "127.31.211.125" port: 43493 } } peers { permanent_uuid: "21e44f6f40f44af3b10c7af4fe478c17" member_type: VOTER last_known_addr { host: "127.31.211.124" port: 42871 } }
02:06:01.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.850355 32744 raft_consensus.cc:383] T 00000000000000000000000000000000 P 2bc5891afdc547f3ba433776f76f9f44 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:06:01.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.850620 32744 raft_consensus.cc:738] T 00000000000000000000000000000000 P 2bc5891afdc547f3ba433776f76f9f44 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2bc5891afdc547f3ba433776f76f9f44, State: Initialized, Role: FOLLOWER
02:06:01.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.851482 32744 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2bc5891afdc547f3ba433776f76f9f44 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5965108c7d9b4918aff8b619718a23d7" member_type: VOTER last_known_addr { host: "127.31.211.126" port: 43055 } } peers { permanent_uuid: "2bc5891afdc547f3ba433776f76f9f44" member_type: VOTER last_known_addr { host: "127.31.211.125" port: 43493 } } peers { permanent_uuid: "21e44f6f40f44af3b10c7af4fe478c17" member_type: VOTER last_known_addr { host: "127.31.211.124" port: 42871 } }
02:06:01.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.868131 379 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2bc5891afdc547f3ba433776f76f9f44 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5965108c7d9b4918aff8b619718a23d7" member_type: VOTER last_known_addr { host: "127.31.211.126" port: 43055 } } peers { permanent_uuid: "2bc5891afdc547f3ba433776f76f9f44" member_type: VOTER last_known_addr { host: "127.31.211.125" port: 43493 } } peers { permanent_uuid: "21e44f6f40f44af3b10c7af4fe478c17" member_type: VOTER last_known_addr { host: "127.31.211.124" port: 42871 } } }
02:06:01.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.870292 32744 sys_catalog.cc:564] T 00000000000000000000000000000000 P 2bc5891afdc547f3ba433776f76f9f44 [sys.catalog]: configured and running, proceeding with master startup.
02:06:01.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.870437 32719 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "21e44f6f40f44af3b10c7af4fe478c17" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2bc5891afdc547f3ba433776f76f9f44" is_pre_election: true
02:06:01.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.871274 32719 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 2bc5891afdc547f3ba433776f76f9f44 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 21e44f6f40f44af3b10c7af4fe478c17 in term 0.
02:06:01.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.869921 379 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2bc5891afdc547f3ba433776f76f9f44 [sys.catalog]: This master's current role is: FOLLOWER
02:06:01.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.872938 307 leader_election.cc:304] T 00000000000000000000000000000000 P 21e44f6f40f44af3b10c7af4fe478c17 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 21e44f6f40f44af3b10c7af4fe478c17, 2bc5891afdc547f3ba433776f76f9f44; no voters:
02:06:01.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.873832 364 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 21e44f6f40f44af3b10c7af4fe478c17 [term 0 FOLLOWER]: Leader pre-election won for term 1
02:06:01.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.874155 364 raft_consensus.cc:491] T 00000000000000000000000000000000 P 21e44f6f40f44af3b10c7af4fe478c17 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:06:01.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.874472 364 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 21e44f6f40f44af3b10c7af4fe478c17 [term 0 FOLLOWER]: Advancing to term 1
02:06:01.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.885687 364 raft_consensus.cc:513] T 00000000000000000000000000000000 P 21e44f6f40f44af3b10c7af4fe478c17 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5965108c7d9b4918aff8b619718a23d7" member_type: VOTER last_known_addr { host: "127.31.211.126" port: 43055 } } peers { permanent_uuid: "2bc5891afdc547f3ba433776f76f9f44" member_type: VOTER last_known_addr { host: "127.31.211.125" port: 43493 } } peers { permanent_uuid: "21e44f6f40f44af3b10c7af4fe478c17" member_type: VOTER last_known_addr { host: "127.31.211.124" port: 42871 } }
02:06:01.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.888494 32719 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "21e44f6f40f44af3b10c7af4fe478c17" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2bc5891afdc547f3ba433776f76f9f44"
02:06:01.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.888734 364 leader_election.cc:290] T 00000000000000000000000000000000 P 21e44f6f40f44af3b10c7af4fe478c17 [CANDIDATE]: Term 1 election: Requested vote from peers 5965108c7d9b4918aff8b619718a23d7 (127.31.211.126:43055), 2bc5891afdc547f3ba433776f76f9f44 (127.31.211.125:43493)
02:06:01.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.889148 32719 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 2bc5891afdc547f3ba433776f76f9f44 [term 0 FOLLOWER]: Advancing to term 1
02:06:01.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.904481 32719 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 2bc5891afdc547f3ba433776f76f9f44 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 21e44f6f40f44af3b10c7af4fe478c17 in term 1.
02:06:01.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.905496 307 leader_election.cc:304] T 00000000000000000000000000000000 P 21e44f6f40f44af3b10c7af4fe478c17 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 21e44f6f40f44af3b10c7af4fe478c17, 2bc5891afdc547f3ba433776f76f9f44; no voters:
02:06:01.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.906292 364 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 21e44f6f40f44af3b10c7af4fe478c17 [term 1 FOLLOWER]: Leader election won for term 1
02:06:01.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.912093 364 raft_consensus.cc:695] T 00000000000000000000000000000000 P 21e44f6f40f44af3b10c7af4fe478c17 [term 1 LEADER]: Becoming Leader. State: Replica: 21e44f6f40f44af3b10c7af4fe478c17, State: Running, Role: LEADER
02:06:01.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.913287 364 consensus_queue.cc:237] T 00000000000000000000000000000000 P 21e44f6f40f44af3b10c7af4fe478c17 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5965108c7d9b4918aff8b619718a23d7" member_type: VOTER last_known_addr { host: "127.31.211.126" port: 43055 } } peers { permanent_uuid: "2bc5891afdc547f3ba433776f76f9f44" member_type: VOTER last_known_addr { host: "127.31.211.125" port: 43493 } } peers { permanent_uuid: "21e44f6f40f44af3b10c7af4fe478c17" member_type: VOTER last_known_addr { host: "127.31.211.124" port: 42871 } }
02:06:01.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:01.921396 391 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 2bc5891afdc547f3ba433776f76f9f44: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:06:01.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:01.921970 391 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:06:01.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.952872 390 sys_catalog.cc:455] T 00000000000000000000000000000000 P 21e44f6f40f44af3b10c7af4fe478c17 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 21e44f6f40f44af3b10c7af4fe478c17. Latest consensus state: current_term: 1 leader_uuid: "21e44f6f40f44af3b10c7af4fe478c17" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5965108c7d9b4918aff8b619718a23d7" member_type: VOTER last_known_addr { host: "127.31.211.126" port: 43055 } } peers { permanent_uuid: "2bc5891afdc547f3ba433776f76f9f44" member_type: VOTER last_known_addr { host: "127.31.211.125" port: 43493 } } peers { permanent_uuid: "21e44f6f40f44af3b10c7af4fe478c17" member_type: VOTER last_known_addr { host: "127.31.211.124" port: 42871 } } }
02:06:01.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.953661 390 sys_catalog.cc:458] T 00000000000000000000000000000000 P 21e44f6f40f44af3b10c7af4fe478c17 [sys.catalog]: This master's current role is: LEADER
02:06:01.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.915627 32671 master_runner.cc:186] Error getting master registration for 127.31.211.126:43055: OK, instance_id { permanent_uuid: "5965108c7d9b4918aff8b619718a23d7" instance_seqno: 1756778757333458 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:06:01.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.958638 32671 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:06:01.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.961546 394 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
02:06:01.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.968778 394 catalog_manager.cc:1486] Initializing Kudu cluster ID...
02:06:01.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.995946 32719 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 2bc5891afdc547f3ba433776f76f9f44 [term 1 FOLLOWER]: Refusing update from remote peer 21e44f6f40f44af3b10c7af4fe478c17: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:06:01.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:01.997779 364 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 21e44f6f40f44af3b10c7af4fe478c17 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2bc5891afdc547f3ba433776f76f9f44" member_type: VOTER last_known_addr { host: "127.31.211.125" port: 43493 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:06:02.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.018956 379 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2bc5891afdc547f3ba433776f76f9f44 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 21e44f6f40f44af3b10c7af4fe478c17. Latest consensus state: current_term: 1 leader_uuid: "21e44f6f40f44af3b10c7af4fe478c17" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5965108c7d9b4918aff8b619718a23d7" member_type: VOTER last_known_addr { host: "127.31.211.126" port: 43055 } } peers { permanent_uuid: "2bc5891afdc547f3ba433776f76f9f44" member_type: VOTER last_known_addr { host: "127.31.211.125" port: 43493 } } peers { permanent_uuid: "21e44f6f40f44af3b10c7af4fe478c17" member_type: VOTER last_known_addr { host: "127.31.211.124" port: 42871 } } }
02:06:02.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.019722 379 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2bc5891afdc547f3ba433776f76f9f44 [sys.catalog]: This master's current role is: FOLLOWER
02:06:02.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.031654 395 mvcc.cc:204] Tried to move back new op lower bound from 7195765809084153856 to 7195765808878657536. Current Snapshot: MvccSnapshot[applied={T|T < 7195765809084153856}]
02:06:02.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.044930 32669 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.211.124" port: 42871 } attempt: 7
02:06:02.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.052937 379 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2bc5891afdc547f3ba433776f76f9f44 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "21e44f6f40f44af3b10c7af4fe478c17" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5965108c7d9b4918aff8b619718a23d7" member_type: VOTER last_known_addr { host: "127.31.211.126" port: 43055 } } peers { permanent_uuid: "2bc5891afdc547f3ba433776f76f9f44" member_type: VOTER last_known_addr { host: "127.31.211.125" port: 43493 } } peers { permanent_uuid: "21e44f6f40f44af3b10c7af4fe478c17" member_type: VOTER last_known_addr { host: "127.31.211.124" port: 42871 } } }
02:06:02.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.053664 379 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2bc5891afdc547f3ba433776f76f9f44 [sys.catalog]: This master's current role is: FOLLOWER
02:06:02.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.069103 364 sys_catalog.cc:455] T 00000000000000000000000000000000 P 21e44f6f40f44af3b10c7af4fe478c17 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "21e44f6f40f44af3b10c7af4fe478c17" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5965108c7d9b4918aff8b619718a23d7" member_type: VOTER last_known_addr { host: "127.31.211.126" port: 43055 } } peers { permanent_uuid: "2bc5891afdc547f3ba433776f76f9f44" member_type: VOTER last_known_addr { host: "127.31.211.125" port: 43493 } } peers { permanent_uuid: "21e44f6f40f44af3b10c7af4fe478c17" member_type: VOTER last_known_addr { host: "127.31.211.124" port: 42871 } } }
02:06:02.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.069828 364 sys_catalog.cc:458] T 00000000000000000000000000000000 P 21e44f6f40f44af3b10c7af4fe478c17 [sys.catalog]: This master's current role is: LEADER
02:06:02.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.071995 394 catalog_manager.cc:1349] Generated new cluster ID: 5048e28a218f40c7a3277ae67820eaff
02:06:02.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.072268 394 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
02:06:02.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.110857 32669 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5965108c7d9b4918aff8b619718a23d7: Bootstrap starting.
02:06:02.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.123347 32669 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 5965108c7d9b4918aff8b619718a23d7: Neither blocks nor log segments found. Creating new log.
02:06:02.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.126045 32669 log.cc:826] T 00000000000000000000000000000000 P 5965108c7d9b4918aff8b619718a23d7: Log is configured to *not* fsync() on all Append() calls
02:06:02.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.126830 394 catalog_manager.cc:1372] Generated new certificate authority record
02:06:02.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.132649 394 catalog_manager.cc:1506] Loading token signing keys...
02:06:02.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.149763 32669 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5965108c7d9b4918aff8b619718a23d7: No bootstrap required, opened a new log
02:06:02.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.186064 32669 raft_consensus.cc:357] T 00000000000000000000000000000000 P 5965108c7d9b4918aff8b619718a23d7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5965108c7d9b4918aff8b619718a23d7" member_type: VOTER last_known_addr { host: "127.31.211.126" port: 43055 } } peers { permanent_uuid: "2bc5891afdc547f3ba433776f76f9f44" member_type: VOTER last_known_addr { host: "127.31.211.125" port: 43493 } } peers { permanent_uuid: "21e44f6f40f44af3b10c7af4fe478c17" member_type: VOTER last_known_addr { host: "127.31.211.124" port: 42871 } }
02:06:02.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.187108 32669 raft_consensus.cc:383] T 00000000000000000000000000000000 P 5965108c7d9b4918aff8b619718a23d7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:06:02.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.187476 32669 raft_consensus.cc:738] T 00000000000000000000000000000000 P 5965108c7d9b4918aff8b619718a23d7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5965108c7d9b4918aff8b619718a23d7, State: Initialized, Role: FOLLOWER
02:06:02.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.188896 32669 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5965108c7d9b4918aff8b619718a23d7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5965108c7d9b4918aff8b619718a23d7" member_type: VOTER last_known_addr { host: "127.31.211.126" port: 43055 } } peers { permanent_uuid: "2bc5891afdc547f3ba433776f76f9f44" member_type: VOTER last_known_addr { host: "127.31.211.125" port: 43493 } } peers { permanent_uuid: "21e44f6f40f44af3b10c7af4fe478c17" member_type: VOTER last_known_addr { host: "127.31.211.124" port: 42871 } }
02:06:02.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.192205 394 catalog_manager.cc:5955] T 00000000000000000000000000000000 P 21e44f6f40f44af3b10c7af4fe478c17: Generated new TSK 0
02:06:02.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.196511 394 catalog_manager.cc:1516] Initializing in-progress tserver states...
02:06:02.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.200800 402 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5965108c7d9b4918aff8b619718a23d7 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5965108c7d9b4918aff8b619718a23d7" member_type: VOTER last_known_addr { host: "127.31.211.126" port: 43055 } } peers { permanent_uuid: "2bc5891afdc547f3ba433776f76f9f44" member_type: VOTER last_known_addr { host: "127.31.211.125" port: 43493 } } peers { permanent_uuid: "21e44f6f40f44af3b10c7af4fe478c17" member_type: VOTER last_known_addr { host: "127.31.211.124" port: 42871 } } }
02:06:02.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.201575 402 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5965108c7d9b4918aff8b619718a23d7 [sys.catalog]: This master's current role is: FOLLOWER
02:06:02.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.202607 32669 sys_catalog.cc:564] T 00000000000000000000000000000000 P 5965108c7d9b4918aff8b619718a23d7 [sys.catalog]: configured and running, proceeding with master startup.
02:06:02.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.203435 32642 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 5965108c7d9b4918aff8b619718a23d7 [term 0 FOLLOWER]: Advancing to term 1
02:06:02.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.207755 32643 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "21e44f6f40f44af3b10c7af4fe478c17" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5965108c7d9b4918aff8b619718a23d7"
02:06:02.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.209220 32644 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "21e44f6f40f44af3b10c7af4fe478c17" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5965108c7d9b4918aff8b619718a23d7" is_pre_election: true
02:06:02.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.214500 32642 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 5965108c7d9b4918aff8b619718a23d7 [term 1 FOLLOWER]: Refusing update from remote peer 21e44f6f40f44af3b10c7af4fe478c17: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:06:02.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.216171 390 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 21e44f6f40f44af3b10c7af4fe478c17 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5965108c7d9b4918aff8b619718a23d7" member_type: VOTER last_known_addr { host: "127.31.211.126" port: 43055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:06:02.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.257239 32589 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu
02:06:02.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu
02:06:02.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-0/wal
02:06:02.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-0/data
02:06:02.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:06:02.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:06:02.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:06:02.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:02.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:02.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:02.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-0/logs
02:06:02.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-0/data/info.pb
02:06:02.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:02.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:06:02.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:06:02.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:06:02.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:06:02.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:02.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:02.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:02.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:06:02.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:06:02.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.211.65:0
02:06:02.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.31.211.65
02:06:02.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.211.65
02:06:02.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:02.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.31.211.126:43055,127.31.211.125:43493,127.31.211.124:42871
02:06:02.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.211.84:35601
02:06:02.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:02.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:02.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:06:02.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.282840 409 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5965108c7d9b4918aff8b619718a23d7 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "21e44f6f40f44af3b10c7af4fe478c17" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5965108c7d9b4918aff8b619718a23d7" member_type: VOTER last_known_addr { host: "127.31.211.126" port: 43055 } } peers { permanent_uuid: "2bc5891afdc547f3ba433776f76f9f44" member_type: VOTER last_known_addr { host: "127.31.211.125" port: 43493 } } peers { permanent_uuid: "21e44f6f40f44af3b10c7af4fe478c17" member_type: VOTER last_known_addr { host: "127.31.211.124" port: 42871 } } }
02:06:02.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.283708 409 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5965108c7d9b4918aff8b619718a23d7 [sys.catalog]: This master's current role is: FOLLOWER
02:06:02.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.283865 402 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5965108c7d9b4918aff8b619718a23d7 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 21e44f6f40f44af3b10c7af4fe478c17. Latest consensus state: current_term: 1 leader_uuid: "21e44f6f40f44af3b10c7af4fe478c17" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5965108c7d9b4918aff8b619718a23d7" member_type: VOTER last_known_addr { host: "127.31.211.126" port: 43055 } } peers { permanent_uuid: "2bc5891afdc547f3ba433776f76f9f44" member_type: VOTER last_known_addr { host: "127.31.211.125" port: 43493 } } peers { permanent_uuid: "21e44f6f40f44af3b10c7af4fe478c17" member_type: VOTER last_known_addr { host: "127.31.211.124" port: 42871 } } }
02:06:02.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.284796 402 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5965108c7d9b4918aff8b619718a23d7 [sys.catalog]: This master's current role is: FOLLOWER
02:06:02.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.313184 419 catalog_manager.cc:1261] Loaded cluster ID: 5048e28a218f40c7a3277ae67820eaff
02:06:02.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.313455 419 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 5965108c7d9b4918aff8b619718a23d7: loading cluster ID for follower catalog manager: success
02:06:02.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.321017 419 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 5965108c7d9b4918aff8b619718a23d7: acquiring CA information for follower catalog manager: success
02:06:02.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.326097 419 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 5965108c7d9b4918aff8b619718a23d7: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:06:02.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.728323 423 sys_catalog.cc:455] T 00000000000000000000000000000000 P 21e44f6f40f44af3b10c7af4fe478c17 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "21e44f6f40f44af3b10c7af4fe478c17" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5965108c7d9b4918aff8b619718a23d7" member_type: VOTER last_known_addr { host: "127.31.211.126" port: 43055 } } peers { permanent_uuid: "2bc5891afdc547f3ba433776f76f9f44" member_type: VOTER last_known_addr { host: "127.31.211.125" port: 43493 } } peers { permanent_uuid: "21e44f6f40f44af3b10c7af4fe478c17" member_type: VOTER last_known_addr { host: "127.31.211.124" port: 42871 } } }
02:06:02.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.729061 423 sys_catalog.cc:458] T 00000000000000000000000000000000 P 21e44f6f40f44af3b10c7af4fe478c17 [sys.catalog]: This master's current role is: LEADER
02:06:02.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:02.807741 412 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:06:02.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:02.809888 412 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:06:02.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:02.810897 412 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:06:02.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:02.869968 412 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:06:02.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:02.871726 412 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.31.211.65
02:06:02.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.928038 391 catalog_manager.cc:1261] Loaded cluster ID: 5048e28a218f40c7a3277ae67820eaff
02:06:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.928706 391 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 2bc5891afdc547f3ba433776f76f9f44: loading cluster ID for follower catalog manager: success
02:06:02.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.937450 412 tablet_server_runner.cc:78] Tablet server non-default flags:
02:06:02.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:02.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.211.84:35601
02:06:02.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:02.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:02.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-0/data
02:06:02.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-0/wal
02:06:02.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:02.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:02.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.211.65:0
02:06:02.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:06:02.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:02.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:02.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-0/data/info.pb
02:06:02.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.211.65
02:06:02.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:02.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.31.211.126:43055,127.31.211.125:43493,127.31.211.124:42871
02:06:02.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:06:02.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:02.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:06:02.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:06:02.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:02.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.31.211.65
02:06:02.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-0/logs
02:06:02.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:02.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:06:02.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:06:02.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:06:02.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:06:02.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
02:06:02.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:06:02.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 01:43:19 UTC on 5fd53c4cbb9d
02:06:02.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7857
02:06:02.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:06:02.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.939249 412 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:06:02.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.941629 412 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:06:02.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.949555 391 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 2bc5891afdc547f3ba433776f76f9f44: acquiring CA information for follower catalog manager: success
02:06:02.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.964726 391 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 2bc5891afdc547f3ba433776f76f9f44: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:06:02.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:02.969099 429 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:04.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:02.984774 412 server_base.cc:1047] running on GCE node
02:06:04.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:02.984223 430 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:04.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:02.983675 433 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:04.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:04.488164 412 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:06:04.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:04.499825 412 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:06:04.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:04.501268 412 hybrid_clock.cc:648] HybridClock initialized: now 1756778764501224 us; error 62 us; skew 500 ppm
02:06:04.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:04.502242 412 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:06:04.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:04.521872 412 webserver.cc:480] Webserver started at http://127.31.211.65:34387/ using document root <none> and password file <none>
02:06:04.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:04.523170 412 fs_manager.cc:362] Metadata directory not provided
02:06:04.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:04.523448 412 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:06:04.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:04.524049 412 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
02:06:04.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:04.530314 412 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-0/data/instance:
02:06:04.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4b8555384043404c8abe001398d17e0f"
02:06:04.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 02:06:04 on dist-test-slave-szd5"
02:06:04.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:04.531739 412 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-0/wal/instance:
02:06:04.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4b8555384043404c8abe001398d17e0f"
02:06:04.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 02:06:04 on dist-test-slave-szd5"
02:06:04.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:04.548044 412 fs_manager.cc:696] Time spent creating directory manager: real 0.016s user 0.013s sys 0.000s
02:06:04.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:04.562368 439 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:06:04.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:04.563597 412 fs_manager.cc:730] Time spent opening block manager: real 0.011s user 0.007s sys 0.003s
02:06:04.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:04.563947 412 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-0/data,/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-0/wal
02:06:04.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4b8555384043404c8abe001398d17e0f"
02:06:04.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 02:06:04 on dist-test-slave-szd5"
02:06:04.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:04.564334 412 fs_report.cc:389] FS layout report
02:06:04.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:06:04.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-0/wal
02:06:04.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-0/wal
02:06:04.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-0/data/data
02:06:04.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:06:04.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:06:04.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:06:04.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:06:04.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:06:04.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:06:04.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:06:04.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:06:04.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:06:04.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:06:04.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:04.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:04.667786 412 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:06:04.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:04.669793 412 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:04.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:04.670348 412 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:04.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:04.680286 412 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:06:04.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:04.692265 412 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:06:04.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:04.692557 412 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:06:04.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:04.692852 412 ts_tablet_manager.cc:610] Registered 0 tablets
02:06:04.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:04.693076 412 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:06:04.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:04.897661 412 rpc_server.cc:307] RPC server started. Bound to: 127.31.211.65:43015
02:06:04.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:04.897765 553 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.31.211.65:43015 every 8 connection(s)
02:06:04.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:04.901731 412 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-0/data/info.pb
02:06:04.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:04.910993 32589 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu as pid 412
02:06:04.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:04.911430 32589 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-0/wal/instance
02:06:04.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:04.925057 32589 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu
02:06:04.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu
02:06:04.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-1/wal
02:06:04.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-1/data
02:06:04.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:06:04.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:06:04.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:06:04.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:04.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:04.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:04.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-1/logs
02:06:04.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-1/data/info.pb
02:06:04.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:04.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:06:04.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:06:04.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:06:04.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:06:04.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:04.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:04.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:04.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:06:04.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:06:04.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.211.66:0
02:06:04.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.31.211.66
02:06:04.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.211.66
02:06:04.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:04.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.31.211.126:43055,127.31.211.125:43493,127.31.211.124:42871
02:06:04.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.211.84:35601
02:06:04.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:04.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:04.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:06:05.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:05.012975 554 heartbeater.cc:344] Connected to a master server at 127.31.211.124:42871
02:06:05.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:05.013626 554 heartbeater.cc:461] Registering TS with master...
02:06:05.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:05.015280 554 heartbeater.cc:507] Master 127.31.211.124:42871 requested a full tablet report, sending...
02:06:05.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:05.019016 321 ts_manager.cc:194] Registered new tserver with Master: 4b8555384043404c8abe001398d17e0f (127.31.211.65:43015)
02:06:05.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:05.021986 321 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.31.211.65:48545
02:06:05.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:05.035811 555 heartbeater.cc:344] Connected to a master server at 127.31.211.125:43493
02:06:05.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:05.036198 555 heartbeater.cc:461] Registering TS with master...
02:06:05.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:05.036857 555 heartbeater.cc:507] Master 127.31.211.125:43493 requested a full tablet report, sending...
02:06:05.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:05.040524 32709 ts_manager.cc:194] Registered new tserver with Master: 4b8555384043404c8abe001398d17e0f (127.31.211.65:43015)
02:06:05.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:05.045087 559 heartbeater.cc:344] Connected to a master server at 127.31.211.126:43055
02:06:05.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:05.045462 559 heartbeater.cc:461] Registering TS with master...
02:06:05.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:05.046125 559 heartbeater.cc:507] Master 127.31.211.126:43055 requested a full tablet report, sending...
02:06:05.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:05.049743 32634 ts_manager.cc:194] Registered new tserver with Master: 4b8555384043404c8abe001398d17e0f (127.31.211.65:43015)
02:06:05.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:05.373034 558 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:06:05.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:05.373554 558 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:06:05.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:05.374589 558 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:06:05.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:05.416951 558 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:06:05.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:05.417918 558 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.31.211.66
02:06:05.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:05.455646 558 tablet_server_runner.cc:78] Tablet server non-default flags:
02:06:05.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:05.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.211.84:35601
02:06:05.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:05.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:05.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-1/data
02:06:05.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-1/wal
02:06:05.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:05.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:05.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.211.66:0
02:06:05.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:06:05.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:05.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:05.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-1/data/info.pb
02:06:05.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.211.66
02:06:05.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:05.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.31.211.126:43055,127.31.211.125:43493,127.31.211.124:42871
02:06:05.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:06:05.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:05.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:06:05.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:06:05.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:05.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.31.211.66
02:06:05.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-1/logs
02:06:05.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:05.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:06:05.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:06:05.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:06:05.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:06:05.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
02:06:05.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:06:05.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 01:43:19 UTC on 5fd53c4cbb9d
02:06:05.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7857
02:06:05.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:06:05.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:05.457234 558 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:06:05.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:05.459414 558 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:06:05.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:05.472606 571 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:06.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:06.026396 554 heartbeater.cc:499] Master 127.31.211.124:42871 was elected leader, sending a full tablet report...
02:06:07.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:06.875825 570 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ea0 after lost signal to thread 558
02:06:07.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:07.224246 558 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.752s user 0.565s sys 1.166s
02:06:07.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:07.224627 558 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.752s user 0.565s sys 1.166s
02:06:07.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:05.472944 572 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:07.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:07.226899 574 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:07.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:07.229229 573 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1753 milliseconds
02:06:07.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:07.229300 558 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:06:07.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:07.230440 558 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:06:07.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:07.232539 558 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:06:07.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:07.233904 558 hybrid_clock.cc:648] HybridClock initialized: now 1756778767233862 us; error 46 us; skew 500 ppm
02:06:07.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:07.234690 558 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:06:07.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:07.240585 558 webserver.cc:480] Webserver started at http://127.31.211.66:38961/ using document root <none> and password file <none>
02:06:07.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:07.241547 558 fs_manager.cc:362] Metadata directory not provided
02:06:07.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:07.241776 558 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:06:07.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:07.242259 558 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
02:06:07.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:07.246515 558 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-1/data/instance:
02:06:07.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "734305aabee54c8098d7cc5d9a6703af"
02:06:07.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 02:06:07 on dist-test-slave-szd5"
02:06:07.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:07.247694 558 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-1/wal/instance:
02:06:07.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "734305aabee54c8098d7cc5d9a6703af"
02:06:07.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 02:06:07 on dist-test-slave-szd5"
02:06:07.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:07.254726 558 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.005s sys 0.001s
02:06:07.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:07.260593 582 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:06:07.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:07.261929 558 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
02:06:07.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:07.262315 558 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-1/data,/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-1/wal
02:06:07.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "734305aabee54c8098d7cc5d9a6703af"
02:06:07.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 02:06:07 on dist-test-slave-szd5"
02:06:07.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:07.262761 558 fs_report.cc:389] FS layout report
02:06:07.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:06:07.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-1/wal
02:06:07.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-1/wal
02:06:07.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-1/data/data
02:06:07.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:06:07.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:06:07.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:06:07.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:06:07.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:06:07.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:06:07.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:06:07.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:06:07.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:06:07.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:06:07.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:07.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:07.319430 558 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:06:07.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:07.320892 558 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:07.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:07.321273 558 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:07.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:07.324218 558 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:06:07.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:07.328166 558 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:06:07.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:07.328349 558 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:06:07.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:07.328534 558 ts_tablet_manager.cc:610] Registered 0 tablets
02:06:07.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:07.328660 558 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:06:07.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:07.489413 558 rpc_server.cc:307] RPC server started. Bound to: 127.31.211.66:41897
02:06:07.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:07.489522 694 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.31.211.66:41897 every 8 connection(s)
02:06:07.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:07.491981 558 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-1/data/info.pb
02:06:07.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:07.496951 32589 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu as pid 558
02:06:07.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:07.497457 32589 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-1/wal/instance
02:06:07.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:07.524353 32589 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu
02:06:07.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu
02:06:07.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-2/wal
02:06:07.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-2/data
02:06:07.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:06:07.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:06:07.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:06:07.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:07.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:07.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:07.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-2/logs
02:06:07.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-2/data/info.pb
02:06:07.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:07.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:06:07.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:06:07.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:06:07.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:06:07.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:07.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:07.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:07.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:06:07.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:06:07.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.211.67:0
02:06:07.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.31.211.67
02:06:07.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.211.67
02:06:07.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:07.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.31.211.126:43055,127.31.211.125:43493,127.31.211.124:42871
02:06:07.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.211.84:35601
02:06:07.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:07.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:07.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:06:07.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:07.563172 696 heartbeater.cc:344] Connected to a master server at 127.31.211.125:43493
02:06:07.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:07.563759 696 heartbeater.cc:461] Registering TS with master...
02:06:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:07.565220 696 heartbeater.cc:507] Master 127.31.211.125:43493 requested a full tablet report, sending...
02:06:07.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:07.568493 32709 ts_manager.cc:194] Registered new tserver with Master: 734305aabee54c8098d7cc5d9a6703af (127.31.211.66:41897)
02:06:07.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:07.571399 697 heartbeater.cc:344] Connected to a master server at 127.31.211.126:43055
02:06:07.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:07.571792 697 heartbeater.cc:461] Registering TS with master...
02:06:07.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:07.572702 697 heartbeater.cc:507] Master 127.31.211.126:43055 requested a full tablet report, sending...
02:06:07.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:07.574115 695 heartbeater.cc:344] Connected to a master server at 127.31.211.124:42871
02:06:07.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:07.574442 695 heartbeater.cc:461] Registering TS with master...
02:06:07.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:07.575315 695 heartbeater.cc:507] Master 127.31.211.124:42871 requested a full tablet report, sending...
02:06:07.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:07.575387 32634 ts_manager.cc:194] Registered new tserver with Master: 734305aabee54c8098d7cc5d9a6703af (127.31.211.66:41897)
02:06:07.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:07.577947 321 ts_manager.cc:194] Registered new tserver with Master: 734305aabee54c8098d7cc5d9a6703af (127.31.211.66:41897)
02:06:07.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:07.579185 321 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.31.211.66:44617
02:06:07.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:07.809762 32665 debug-util.cc:398] Leaking SignalData structure 0x7b080009c500 after lost signal to thread 32602
02:06:07.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:07.867619 705 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:06:07.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:07.868180 705 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:06:07.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:07.869134 705 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:06:07.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:07.906996 705 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:06:07.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:07.908423 705 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.31.211.67
02:06:07.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:07.943687 705 tablet_server_runner.cc:78] Tablet server non-default flags:
02:06:07.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:07.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.211.84:35601
02:06:07.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:07.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:07.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-2/data
02:06:07.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-2/wal
02:06:07.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:07.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:07.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.211.67:0
02:06:07.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:06:07.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:07.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:07.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-2/data/info.pb
02:06:07.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.211.67
02:06:07.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:07.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.31.211.126:43055,127.31.211.125:43493,127.31.211.124:42871
02:06:07.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:06:07.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:07.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:06:07.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:06:07.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:07.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.31.211.67
02:06:07.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-2/logs
02:06:07.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:07.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:06:07.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:06:07.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:06:07.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:06:07.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
02:06:07.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:06:07.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 01:43:19 UTC on 5fd53c4cbb9d
02:06:07.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7857
02:06:07.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:06:07.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:07.945036 705 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:06:07.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:07.946801 705 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:06:07.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:07.960273 713 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:08.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:08.581965 695 heartbeater.cc:499] Master 127.31.211.124:42871 was elected leader, sending a full tablet report...
02:06:09.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:07.964042 716 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:09.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:07.962266 714 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:09.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:09.356612 715 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1391 milliseconds
02:06:09.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:09.356719 705 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:06:09.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:09.358188 705 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:06:09.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:09.361038 705 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:06:09.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:09.362463 705 hybrid_clock.cc:648] HybridClock initialized: now 1756778769362398 us; error 69 us; skew 500 ppm
02:06:09.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:09.363170 705 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:06:09.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:09.371212 705 webserver.cc:480] Webserver started at http://127.31.211.67:39267/ using document root <none> and password file <none>
02:06:09.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:09.372498 705 fs_manager.cc:362] Metadata directory not provided
02:06:09.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:09.372766 705 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:06:09.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:09.373265 705 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
02:06:09.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:09.377542 705 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-2/data/instance:
02:06:09.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ad1da09105d24327b4f0ce9b2bc5a365"
02:06:09.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 02:06:09 on dist-test-slave-szd5"
02:06:09.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:09.378661 705 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-2/wal/instance:
02:06:09.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ad1da09105d24327b4f0ce9b2bc5a365"
02:06:09.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 02:06:09 on dist-test-slave-szd5"
02:06:09.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:09.385903 705 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
02:06:09.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:09.392194 724 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:06:09.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:09.393416 705 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
02:06:09.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:09.393813 705 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-2/data,/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-2/wal
02:06:09.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ad1da09105d24327b4f0ce9b2bc5a365"
02:06:09.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 02:06:09 on dist-test-slave-szd5"
02:06:09.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:09.394305 705 fs_report.cc:389] FS layout report
02:06:09.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:06:09.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-2/wal
02:06:09.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-2/wal
02:06:09.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-2/data/data
02:06:09.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:06:09.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:06:09.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:06:09.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:06:09.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:06:09.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:06:09.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:06:09.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:06:09.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:06:09.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:06:09.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:09.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:09.462047 705 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:06:09.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:09.463797 705 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:09.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:09.464185 705 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:09.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:09.467342 705 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:06:09.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:09.472834 705 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:06:09.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:09.473043 705 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:06:09.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:09.473227 705 ts_tablet_manager.cc:610] Registered 0 tablets
02:06:09.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:09.473357 705 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:06:09.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:09.673359 705 rpc_server.cc:307] RPC server started. Bound to: 127.31.211.67:40059
02:06:09.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:09.673542 836 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.31.211.67:40059 every 8 connection(s)
02:06:09.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:09.675954 705 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-2/data/info.pb
02:06:09.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:09.679405 32589 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu as pid 705
02:06:09.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:09.679888 32589 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster3360772805694469395/ts-2/wal/instance
02:06:09.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:09.728348 838 heartbeater.cc:344] Connected to a master server at 127.31.211.125:43493
02:06:09.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:09.728796 838 heartbeater.cc:461] Registering TS with master...
02:06:09.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:09.729792 838 heartbeater.cc:507] Master 127.31.211.125:43493 requested a full tablet report, sending...
02:06:09.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:09.732208 32709 ts_manager.cc:194] Registered new tserver with Master: ad1da09105d24327b4f0ce9b2bc5a365 (127.31.211.67:40059)
02:06:09.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:09.745975 839 heartbeater.cc:344] Connected to a master server at 127.31.211.126:43055
02:06:09.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:09.746331 839 heartbeater.cc:461] Registering TS with master...
02:06:09.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:09.747130 839 heartbeater.cc:507] Master 127.31.211.126:43055 requested a full tablet report, sending...
02:06:09.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:09.749207 32634 ts_manager.cc:194] Registered new tserver with Master: ad1da09105d24327b4f0ce9b2bc5a365 (127.31.211.67:40059)
02:06:09.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:09.750555 837 heartbeater.cc:344] Connected to a master server at 127.31.211.124:42871
02:06:09.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:09.750931 837 heartbeater.cc:461] Registering TS with master...
02:06:09.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:09.751737 837 heartbeater.cc:507] Master 127.31.211.124:42871 requested a full tablet report, sending...
02:06:09.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:09.753765 321 ts_manager.cc:194] Registered new tserver with Master: ad1da09105d24327b4f0ce9b2bc5a365 (127.31.211.67:40059)
02:06:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:09.755210 321 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.31.211.67:35145
02:06:09.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:09.767251 32589 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
02:06:09.777 [DEBUG - main] (MiniKuduCluster.java:195) Response:
02:06:09.780 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
02:06:09.790 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.31.211.126"
port: 43055
}
bound_http_address {
host: "127.31.211.126"
port: 39133
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.31.211.125"
port: 43493
}
bound_http_address {
host: "127.31.211.125"
port: 38061
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.31.211.124"
port: 42871
}
bound_http_address {
host: "127.31.211.124"
port: 40867
}
}
}
02:06:09.793 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
02:06:09.796 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.31.211.65"
port: 43015
}
bound_http_address {
host: "127.31.211.65"
port: 34387
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.31.211.66"
port: 41897
}
bound_http_address {
host: "127.31.211.66"
port: 38961
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.31.211.67"
port: 40059
}
bound_http_address {
host: "127.31.211.67"
port: 39267
}
}
}
02:06:09.796 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
02:06:09.976 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_275]
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51) ~[main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52) [main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41) [main/:?]
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43) [main/:?]
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155) [main/:?]
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138) [main/:?]
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431) [main/:?]
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:299) [main/:?]
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109) [main/:?]
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50) [junit-4.13.2.jar:4.13.2]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:128) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:27) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:137) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:115) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.main(JUnitCore.java:36) [junit-4.13.2.jar:4.13.2]
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.lang.Class.getDeclaredMethod(Class.java:2130)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
02:06:10.005 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
02:06:10.165 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskONRpAo/test-tmp
02:06:10.166 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
02:06:10.166 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskONRpAo/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
02:06:10.169 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
02:06:10.481 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:06:10.481 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
02:06:10.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:10.484388 851 mini_oidc.cc:150] Starting JWKS server
02:06:10.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:10.488282 851 webserver.cc:480] Webserver started at http://127.0.0.1:38193/ using document root <none> and password file <none>
02:06:10.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:10.488754 851 mini_oidc.cc:186] Starting OIDC Discovery server
02:06:10.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:10.490278 851 webserver.cc:480] Webserver started at http://127.0.0.1:35813/ using document root <none> and password file <none>
02:06:10.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
02:06:10.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/principal' for realm 'KRBTEST.COM',
02:06:10.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
02:06:10.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:10 dist-test-slave-szd5 krb5kdc[863](info): setting up network...
02:06:10.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
02:06:10.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:10 dist-test-slave-szd5 krb5kdc[863](info): set up 2 sockets
02:06:10.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:10 dist-test-slave-szd5 krb5kdc[863](info): commencing operation
02:06:10.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
02:06:10.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:10.758073 837 heartbeater.cc:499] Master 127.31.211.124:42871 was elected leader, sending a full tablet report...
02:06:11.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:11.101002 691 debug-util.cc:398] Leaking SignalData structure 0x7b08000acaa0 after lost signal to thread 565
02:06:11.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:11.101876 691 debug-util.cc:398] Leaking SignalData structure 0x7b08000acd20 after lost signal to thread 694
02:06:12.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:12.598007 851 mini_kdc.cc:121] Time spent starting KDC: real 2.073s user 0.004s sys 0.015s
02:06:12.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
02:06:12.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:06:12.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
02:06:12.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
02:06:12.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:06:12.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
02:06:12.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
02:06:12.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:06:12.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
02:06:12.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:06:12.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/test-user.keytab.
02:06:12.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/test-user.keytab.
02:06:12.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:12 dist-test-slave-szd5 krb5kdc[863](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778772, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:12.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
02:06:12.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-02T02:06:12Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
02:06:12.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-02T02:06:12Z Disabled control of system clock
02:06:12.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.0.212.254@KRBTEST.COM; defaulting to no policy
02:06:12.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:06:12.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.0.212.254@KRBTEST.COM" created.
02:06:12.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:06:12.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.0.212.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab.
02:06:12.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.0.212.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab.
02:06:12.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.0.212.254@KRBTEST.COM; defaulting to no policy
02:06:12.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:06:12.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.0.212.254@KRBTEST.COM" created.
02:06:12.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:06:12.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.0.212.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab.
02:06:12.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.0.212.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab.
02:06:12.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:12.837675 851 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu
02:06:12.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu
02:06:12.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/wal
02:06:12.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/data
02:06:12.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:06:12.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:06:12.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:06:12.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:12.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:12.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:12.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/logs
02:06:12.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/data/info.pb
02:06:12.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:12.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:06:12.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:06:12.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:06:12.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:06:12.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:12.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:12.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:12.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:06:12.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:06:12.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:12.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:12.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.212.254:46217
02:06:12.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.212.254
02:06:12.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:12.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab
02:06:12.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.212.254
02:06:12.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:12.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:06:12.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:06:12.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:06:12.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.212.212:38213
02:06:12.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:12.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:12.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:12.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:12.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.212.254:46217,127.0.212.253:32829,127.0.212.252:41741
02:06:12.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:12.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38193/jwks.json
02:06:12.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:06:12.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:06:13.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:13.212782 880 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:06:13.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:13.213388 880 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:06:13.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:13.214007 880 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:06:13.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:13.246275 880 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:06:13.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:13.246524 880 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:06:13.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:13.246806 880 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:06:13.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:13.247021 880 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:06:13.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:13.247212 880 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:06:13.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:13.247396 880 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:06:13.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:13.247588 880 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38193/jwks.json
02:06:13.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:13.284961 880 master_runner.cc:386] Master server non-default flags:
02:06:13.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:13.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.212.212:38213
02:06:13.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:13.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:13.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/data
02:06:13.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/wal
02:06:13.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:06:13.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:13.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.212.254:46217,127.0.212.253:32829,127.0.212.252:41741
02:06:13.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab
02:06:13.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.212.254
02:06:13.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:13.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:13.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:13.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.212.254:46217
02:06:13.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:13.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:06:13.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:13.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38193/jwks.json
02:06:13.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:13.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:06:13.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:06:13.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:13.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:13.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/data/info.pb
02:06:13.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.212.254
02:06:13.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:13.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:06:13.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:13.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:06:13.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:06:13.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:13.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/logs
02:06:13.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:13.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:06:13.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:06:13.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:06:13.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:06:13.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
02:06:13.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:06:13.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 01:43:19 UTC on 5fd53c4cbb9d
02:06:13.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7857
02:06:13.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:06:13.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:13.286207 880 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:06:13.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:13.287165 880 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:06:13.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:13.301204 886 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:13.304467 887 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:14.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:14.646983 880 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.340s user 0.000s sys 0.010s
02:06:14.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:14.647329 880 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.340s user 0.000s sys 0.010s
02:06:14.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:14.649835 889 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:14.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:14.651120 880 server_base.cc:1047] running on GCE node
02:06:14.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:14.652369 880 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:06:14.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:14.655591 880 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:06:14.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:14.656951 880 hybrid_clock.cc:648] HybridClock initialized: now 1756778774656905 us; error 53 us; skew 500 ppm
02:06:14.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:14.657827 880 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:06:14.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:14 dist-test-slave-szd5 krb5kdc[863](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778774, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:14.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:14.666817 880 init.cc:377] Logged in from keytab as kudu/127.0.212.254@KRBTEST.COM (short username kudu)
02:06:14.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:14.677402 880 webserver.cc:480] Webserver started at http://127.0.212.254:44809/ using document root <none> and password file <none>
02:06:14.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:14.678588 880 fs_manager.cc:362] Metadata directory not provided
02:06:14.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:14.678861 880 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:06:14.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:14.679414 880 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
02:06:14.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:14.686239 880 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/data/instance:
02:06:14.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d068f217170340f9b4955443ddfda35f"
02:06:14.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 02:06:14 on dist-test-slave-szd5"
02:06:14.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:14.687526 880 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/wal/instance:
02:06:14.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d068f217170340f9b4955443ddfda35f"
02:06:14.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 02:06:14 on dist-test-slave-szd5"
02:06:14.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:14.699054 880 fs_manager.cc:696] Time spent creating directory manager: real 0.011s user 0.005s sys 0.005s
02:06:14.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:14.709079 897 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:06:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:14.710688 880 fs_manager.cc:730] Time spent opening block manager: real 0.008s user 0.004s sys 0.005s
02:06:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:14.711102 880 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/data,/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/wal
02:06:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d068f217170340f9b4955443ddfda35f"
02:06:14.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 02:06:14 on dist-test-slave-szd5"
02:06:14.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:14.711411 880 fs_report.cc:389] FS layout report
02:06:14.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:06:14.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/wal
02:06:14.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/wal
02:06:14.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/data/data
02:06:14.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:06:14.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:06:14.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:06:14.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:06:14.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:06:14.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:06:14.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:06:14.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:06:14.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:06:14.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:06:14.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:14.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:14.794306 880 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:06:14.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:14.800942 880 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:14.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:14.801648 880 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:14.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:14.976020 880 rpc_server.cc:307] RPC server started. Bound to: 127.0.212.254:46217
02:06:14.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:14.976200 948 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.212.254:46217 every 8 connection(s)
02:06:14.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:14.978755 880 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/data/info.pb
02:06:14.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:14.981472 851 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu as pid 880
02:06:14.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:14.982106 851 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/wal/instance
02:06:14.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:14.989663 949 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:06:15.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.0.212.253@KRBTEST.COM; defaulting to no policy
02:06:15.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:15.006052 949 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } has no permanent_uuid. Determining permanent_uuid...
02:06:15.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:06:15.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.0.212.253@KRBTEST.COM" created.
02:06:15.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:06:15.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.0.212.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab.
02:06:15.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.0.212.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab.
02:06:15.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:15 dist-test-slave-szd5 krb5kdc[863](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778774, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.254@KRBTEST.COM for kudu/127.0.212.254@KRBTEST.COM
02:06:15.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.0.212.253@KRBTEST.COM; defaulting to no policy
02:06:15.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:06:15.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.0.212.253@KRBTEST.COM" created.
02:06:15.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:15.058019 949 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } has no permanent_uuid. Determining permanent_uuid...
02:06:15.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:15.062049 901 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.212.253:32829: connect: Connection refused (error 111)
02:06:15.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:15.071254 949 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.212.253:32829: Network error: Client connection negotiation failed: client connection to 127.0.212.253:32829: connect: Connection refused (error 111)
02:06:15.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:06:15.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.0.212.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab.
02:06:15.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.0.212.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab.
02:06:15.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:15.084725 851 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu
02:06:15.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu
02:06:15.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/wal
02:06:15.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/data
02:06:15.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:06:15.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:06:15.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:06:15.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:15.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:15.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:15.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/logs
02:06:15.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/data/info.pb
02:06:15.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:15.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:06:15.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:06:15.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:06:15.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:06:15.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:15.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:15.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:15.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:06:15.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:06:15.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:15.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:15.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.212.253:32829
02:06:15.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.212.253
02:06:15.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:15.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab
02:06:15.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.212.253
02:06:15.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:15.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:06:15.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:06:15.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:06:15.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.212.212:38213
02:06:15.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:15.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:15.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:15.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:15.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.212.254:46217,127.0.212.253:32829,127.0.212.252:41741
02:06:15.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:15.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38193/jwks.json
02:06:15.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:06:15.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:06:15.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:15.120925 949 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } attempt: 1
02:06:15.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:15.125938 949 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.212.253:32829: Network error: Client connection negotiation failed: client connection to 127.0.212.253:32829: connect: Connection refused (error 111)
02:06:15.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:15.194311 949 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } attempt: 2
02:06:15.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:15.198998 949 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.212.253:32829: Network error: Client connection negotiation failed: client connection to 127.0.212.253:32829: connect: Connection refused (error 111)
02:06:15.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:15.290417 949 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } attempt: 3
02:06:15.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:15.295750 949 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.212.253:32829: Network error: Client connection negotiation failed: client connection to 127.0.212.253:32829: connect: Connection refused (error 111)
02:06:15.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:15.439333 949 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } attempt: 4
02:06:15.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:15.443678 949 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.212.253:32829: Network error: Client connection negotiation failed: client connection to 127.0.212.253:32829: connect: Connection refused (error 111)
02:06:15.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:15.503998 958 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:06:15.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:15.504590 958 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:06:15.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:15.505213 958 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:06:15.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:15.538278 958 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:06:15.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:15.538587 958 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:06:15.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:15.538986 958 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:06:15.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:15.539292 958 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:06:15.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:15.539671 958 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:06:15.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:15.539970 958 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:06:15.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:15.540185 958 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38193/jwks.json
02:06:15.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:15.578833 958 master_runner.cc:386] Master server non-default flags:
02:06:15.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:15.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.212.212:38213
02:06:15.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:15.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:15.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/data
02:06:15.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/wal
02:06:15.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:06:15.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:15.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.212.254:46217,127.0.212.253:32829,127.0.212.252:41741
02:06:15.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab
02:06:15.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.212.253
02:06:15.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:15.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:15.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:15.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.212.253:32829
02:06:15.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:15.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:06:15.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:15.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38193/jwks.json
02:06:15.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:15.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:06:15.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:06:15.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:15.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:15.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/data/info.pb
02:06:15.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.212.253
02:06:15.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:15.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:06:15.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:15.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:06:15.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:06:15.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:15.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/logs
02:06:15.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:15.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:06:15.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:06:15.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:06:15.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:06:15.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
02:06:15.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:06:15.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 01:43:19 UTC on 5fd53c4cbb9d
02:06:15.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7857
02:06:15.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:06:15.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:15.580542 958 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:06:15.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:15.581692 958 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:06:15.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:15.593194 967 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:15.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:15.743069 949 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } attempt: 5
02:06:15.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:15.750586 949 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.212.253:32829: Network error: Client connection negotiation failed: client connection to 127.0.212.253:32829: connect: Connection refused (error 111)
02:06:16.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:16.298146 949 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } attempt: 6
02:06:16.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:16.333309 949 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.212.253:32829: Network error: Client connection negotiation failed: client connection to 127.0.212.253:32829: connect: Connection refused (error 111)
02:06:17.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:16.996290 966 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 958
02:06:17.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:17.377852 958 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.785s user 0.597s sys 1.117s
02:06:17.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:17.378198 958 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.785s user 0.597s sys 1.117s
02:06:17.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:15.593847 968 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:17.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:17.380713 969 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1785 milliseconds
02:06:17.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:17.381563 958 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:06:17.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:17.381922 970 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:17.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:17.386822 958 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:06:17.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:17.390838 958 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:06:17.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:17.392211 958 hybrid_clock.cc:648] HybridClock initialized: now 1756778777392174 us; error 48 us; skew 500 ppm
02:06:17.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:17.393003 958 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:06:17.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:17.393822 949 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } attempt: 7
02:06:17.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:17.401665 949 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.212.253:32829: Network error: Client connection negotiation failed: client connection to 127.0.212.253:32829: connect: Connection refused (error 111)
02:06:17.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:17 dist-test-slave-szd5 krb5kdc[863](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778777, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:17.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:17.402905 958 init.cc:377] Logged in from keytab as kudu/127.0.212.253@KRBTEST.COM (short username kudu)
02:06:17.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:17.411338 958 webserver.cc:480] Webserver started at http://127.0.212.253:39491/ using document root <none> and password file <none>
02:06:17.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:17.412863 958 fs_manager.cc:362] Metadata directory not provided
02:06:17.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:17.413112 958 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:06:17.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:17.413657 958 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
02:06:17.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:17.420361 958 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/data/instance:
02:06:17.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ced15ce4e97b4e92baa9efefcecd6a36"
02:06:17.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 02:06:17 on dist-test-slave-szd5"
02:06:17.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:17.421922 958 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/wal/instance:
02:06:17.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ced15ce4e97b4e92baa9efefcecd6a36"
02:06:17.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 02:06:17 on dist-test-slave-szd5"
02:06:17.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:17.433598 958 fs_manager.cc:696] Time spent creating directory manager: real 0.011s user 0.007s sys 0.006s
02:06:17.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:17.441967 983 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:06:17.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:17.443221 958 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.002s sys 0.004s
02:06:17.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:17.443511 958 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/data,/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/wal
02:06:17.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ced15ce4e97b4e92baa9efefcecd6a36"
02:06:17.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 02:06:17 on dist-test-slave-szd5"
02:06:17.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:17.443969 958 fs_report.cc:389] FS layout report
02:06:17.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:06:17.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/wal
02:06:17.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/wal
02:06:17.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/data/data
02:06:17.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:06:17.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:06:17.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:06:17.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:06:17.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:06:17.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:06:17.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:06:17.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:06:17.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:06:17.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:06:17.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:17.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:17.510571 958 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:06:17.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:17.522410 958 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:17.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:17.523018 958 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:17.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:17.693612 958 rpc_server.cc:307] RPC server started. Bound to: 127.0.212.253:32829
02:06:17.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:17.693799 1034 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.212.253:32829 every 8 connection(s)
02:06:17.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:17.696458 958 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/data/info.pb
02:06:17.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:17.702203 851 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu as pid 958
02:06:17.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:17.702752 851 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/wal/instance
02:06:17.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:17.705883 1036 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:06:17.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:17.717298 1036 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } has no permanent_uuid. Determining permanent_uuid...
02:06:17.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.0.212.252@KRBTEST.COM; defaulting to no policy
02:06:17.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:06:17.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.0.212.252@KRBTEST.COM" created.
02:06:17.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:17 dist-test-slave-szd5 krb5kdc[863](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778777, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.253@KRBTEST.COM for kudu/127.0.212.254@KRBTEST.COM
02:06:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:06:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.0.212.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab.
02:06:17.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.0.212.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab.
02:06:17.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:17.759613 1036 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } has no permanent_uuid. Determining permanent_uuid...
02:06:17.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.0.212.252@KRBTEST.COM; defaulting to no policy
02:06:17.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:06:17.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.0.212.252@KRBTEST.COM" created.
02:06:17.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:17 dist-test-slave-szd5 krb5kdc[863](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778777, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.253@KRBTEST.COM for kudu/127.0.212.253@KRBTEST.COM
02:06:17.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:17.800661 1036 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } has no permanent_uuid. Determining permanent_uuid...
02:06:17.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:06:17.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.0.212.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab.
02:06:17.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.0.212.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab.
02:06:17.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:17.803224 984 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.212.252:41741: connect: Connection refused (error 111)
02:06:17.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:17.803488 851 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu
02:06:17.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu
02:06:17.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/wal
02:06:17.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/data
02:06:17.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:06:17.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:06:17.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:06:17.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:17.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:17.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:17.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/logs
02:06:17.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/data/info.pb
02:06:17.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:17.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:06:17.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:06:17.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:06:17.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:06:17.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:17.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:17.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:17.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:06:17.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:06:17.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:17.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:17.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.212.252:41741
02:06:17.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.212.252
02:06:17.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:17.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab
02:06:17.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.212.252
02:06:17.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:17.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:06:17.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:06:17.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:06:17.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.212.212:38213
02:06:17.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:17.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:17.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:17.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:17.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.212.254:46217,127.0.212.253:32829,127.0.212.252:41741
02:06:17.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:17.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38193/jwks.json
02:06:17.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:06:17.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:06:17.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:17.810101 1036 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.212.252:41741: Network error: Client connection negotiation failed: client connection to 127.0.212.252:41741: connect: Connection refused (error 111)
02:06:17.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:17.859799 1036 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } attempt: 1
02:06:17.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:17.863476 1036 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.212.252:41741: Network error: Client connection negotiation failed: client connection to 127.0.212.252:41741: connect: Connection refused (error 111)
02:06:17.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:17.931937 1036 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } attempt: 2
02:06:17.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:17.935791 1036 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.212.252:41741: Network error: Client connection negotiation failed: client connection to 127.0.212.252:41741: connect: Connection refused (error 111)
02:06:18.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:18.027261 1036 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } attempt: 3
02:06:18.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:18.032419 1036 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.212.252:41741: Network error: Client connection negotiation failed: client connection to 127.0.212.252:41741: connect: Connection refused (error 111)
02:06:18.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:18.175876 1036 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } attempt: 4
02:06:18.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:18.181289 1046 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:06:18.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:18.181691 1036 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.212.252:41741: Network error: Client connection negotiation failed: client connection to 127.0.212.252:41741: connect: Connection refused (error 111)
02:06:18.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:18.182587 1046 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:06:18.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:18.183761 1046 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:06:18.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:18.216578 1046 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:06:18.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:18.216820 1046 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:06:18.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:18.217083 1046 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:06:18.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:18.217279 1046 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:06:18.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:18.217478 1046 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:06:18.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:18.217656 1046 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:06:18.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:18.217800 1046 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38193/jwks.json
02:06:18.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:18.256372 1046 master_runner.cc:386] Master server non-default flags:
02:06:18.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:18.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.212.212:38213
02:06:18.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:18.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:18.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/data
02:06:18.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/wal
02:06:18.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:06:18.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:18.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.212.254:46217,127.0.212.253:32829,127.0.212.252:41741
02:06:18.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab
02:06:18.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.212.252
02:06:18.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:18.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:18.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:18.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.212.252:41741
02:06:18.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:18.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:06:18.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:18.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38193/jwks.json
02:06:18.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:18.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:06:18.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:06:18.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:18.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:18.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/data/info.pb
02:06:18.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.212.252
02:06:18.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:18.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:06:18.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:18.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:06:18.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:06:18.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:18.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/logs
02:06:18.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:18.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:06:18.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:06:18.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:06:18.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:06:18.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
02:06:18.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:06:18.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 01:43:19 UTC on 5fd53c4cbb9d
02:06:18.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7857
02:06:18.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:06:18.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:18.257639 1046 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:06:18.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:18.258613 1046 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:06:18.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:18.273326 1053 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:18.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:18.481132 1036 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } attempt: 5
02:06:18.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:18.490100 1036 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.212.252:41741: Network error: Client connection negotiation failed: client connection to 127.0.212.252:41741: connect: Connection refused (error 111)
02:06:19.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:19.037729 1036 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } attempt: 6
02:06:19.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:19.054574 1036 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.212.252:41741: Network error: Client connection negotiation failed: client connection to 127.0.212.252:41741: connect: Connection refused (error 111)
02:06:19.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:19.492309 949 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } attempt: 8
02:06:19.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:19 dist-test-slave-szd5 krb5kdc[863](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778774, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.254@KRBTEST.COM for kudu/127.0.212.253@KRBTEST.COM
02:06:19.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:19.591677 949 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } has no permanent_uuid. Determining permanent_uuid...
02:06:19.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:19.619277 949 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.212.252:41741: Network error: Client connection negotiation failed: client connection to 127.0.212.252:41741: connect: Connection refused (error 111)
02:06:19.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:19.685411 949 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } attempt: 1
02:06:19.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:19.697237 949 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.212.252:41741: Network error: Client connection negotiation failed: client connection to 127.0.212.252:41741: connect: Connection refused (error 111)
02:06:19.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:19.751616 949 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } attempt: 2
02:06:19.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:19.768745 949 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.212.252:41741: Network error: Client connection negotiation failed: client connection to 127.0.212.252:41741: connect: Connection refused (error 111)
02:06:19.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:19.846752 949 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } attempt: 3
02:06:19.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:19.859645 949 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.212.252:41741: Network error: Client connection negotiation failed: client connection to 127.0.212.252:41741: connect: Connection refused (error 111)
02:06:20.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:20.015753 949 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } attempt: 4
02:06:20.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:20.033596 949 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.212.252:41741: Network error: Client connection negotiation failed: client connection to 127.0.212.252:41741: connect: Connection refused (error 111)
02:06:20.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:20.115852 1036 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } attempt: 7
02:06:20.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:20.169622 1036 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.212.252:41741: Network error: Client connection negotiation failed: client connection to 127.0.212.252:41741: connect: Connection refused (error 111)
02:06:20.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:20.330710 949 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } attempt: 5
02:06:20.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:20.336670 898 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.212.252:41741: connect: Connection refused (error 111) [suppressed 12 similar messages]
02:06:20.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:20.343348 949 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.212.252:41741: Network error: Client connection negotiation failed: client connection to 127.0.212.252:41741: connect: Connection refused (error 111)
02:06:20.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:18.275945 1054 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:20.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:19.681216 1052 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 1046
02:06:20.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:20.353667 1052 kernel_stack_watchdog.cc:198] Thread 1046 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 397ms:
02:06:20.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:06:20.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:06:20.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:06:20.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:06:20.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:20.371688 1046 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.093s user 0.017s sys 0.005s
02:06:20.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:20.374775 1046 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.097s user 0.020s sys 0.005s
02:06:20.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:20.380776 1056 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:20.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:20.383044 1046 server_base.cc:1047] running on GCE node
02:06:20.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:20.387234 1046 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:06:20.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:20.394016 1046 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:06:20.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:20.397864 1046 hybrid_clock.cc:648] HybridClock initialized: now 1756778780397780 us; error 115 us; skew 500 ppm
02:06:20.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:20.400110 1046 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:06:20.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:20 dist-test-slave-szd5 krb5kdc[863](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778780, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:20.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:20.420217 1046 init.cc:377] Logged in from keytab as kudu/127.0.212.252@KRBTEST.COM (short username kudu)
02:06:20.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:20.445716 1046 webserver.cc:480] Webserver started at http://127.0.212.252:33899/ using document root <none> and password file <none>
02:06:20.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:20.448287 1046 fs_manager.cc:362] Metadata directory not provided
02:06:20.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:20.448872 1046 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:06:20.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:20.450098 1046 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
02:06:20.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:20.462489 1046 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/data/instance:
02:06:20.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bdab70b6529844f5a3f4d6f008ba1079"
02:06:20.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 02:06:20 on dist-test-slave-szd5"
02:06:20.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:20.465049 1046 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/wal/instance:
02:06:20.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bdab70b6529844f5a3f4d6f008ba1079"
02:06:20.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 02:06:20 on dist-test-slave-szd5"
02:06:20.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:20.482834 1046 fs_manager.cc:696] Time spent creating directory manager: real 0.017s user 0.010s sys 0.005s
02:06:20.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:20.496491 1074 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:06:20.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:20.498567 1046 fs_manager.cc:730] Time spent opening block manager: real 0.007s user 0.004s sys 0.003s
02:06:20.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:20.499338 1046 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/data,/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/wal
02:06:20.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bdab70b6529844f5a3f4d6f008ba1079"
02:06:20.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 02:06:20 on dist-test-slave-szd5"
02:06:20.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:20.500082 1046 fs_report.cc:389] FS layout report
02:06:20.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:06:20.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/wal
02:06:20.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/wal
02:06:20.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/data/data
02:06:20.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:06:20.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:06:20.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:06:20.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:06:20.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:06:20.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:06:20.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:06:20.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:06:20.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:06:20.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:06:20.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:20.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:20.626674 1046 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:06:20.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:20.637634 1046 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:20.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:20.638763 1046 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:20.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:20.850152 1046 rpc_server.cc:307] RPC server started. Bound to: 127.0.212.252:41741
02:06:20.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:20.850296 1125 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.212.252:41741 every 8 connection(s)
02:06:20.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:20.853108 1046 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/data/info.pb
02:06:20.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:20.861888 851 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu as pid 1046
02:06:20.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:20.862543 851 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/wal/instance
02:06:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:20.865296 1126 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:06:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:20.865372 949 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } attempt: 6
02:06:20.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:20.901252 1126 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } has no permanent_uuid. Determining permanent_uuid...
02:06:20.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:20 dist-test-slave-szd5 krb5kdc[863](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778774, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.254@KRBTEST.COM for kudu/127.0.212.252@KRBTEST.COM
02:06:20.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:20 dist-test-slave-szd5 krb5kdc[863](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778772, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.0.212.254@KRBTEST.COM
02:06:20.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:20 dist-test-slave-szd5 krb5kdc[863](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778780, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.252@KRBTEST.COM for kudu/127.0.212.254@KRBTEST.COM
02:06:20.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:20.957049 949 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f: Bootstrap starting.
02:06:20.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:20.969615 949 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f: Neither blocks nor log segments found. Creating new log.
02:06:20.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:20 dist-test-slave-szd5 krb5kdc[863](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778772, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.0.212.253@KRBTEST.COM
02:06:20.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:20.974177 949 log.cc:826] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f: Log is configured to *not* fsync() on all Append() calls
02:06:20.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:20.976711 1126 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } has no permanent_uuid. Determining permanent_uuid...
02:06:20.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:20.983453 949 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f: No bootstrap required, opened a new log
02:06:21.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:21 dist-test-slave-szd5 krb5kdc[863](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778780, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.252@KRBTEST.COM for kudu/127.0.212.253@KRBTEST.COM
02:06:21.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:21 dist-test-slave-szd5 krb5kdc[863](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778772, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.0.212.252@KRBTEST.COM
02:06:21.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.027436 949 raft_consensus.cc:357] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } }
02:06:21.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.028601 949 raft_consensus.cc:383] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:06:21.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.028952 949 raft_consensus.cc:738] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d068f217170340f9b4955443ddfda35f, State: Initialized, Role: FOLLOWER
02:06:21.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.030534 949 consensus_queue.cc:260] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } }
02:06:21.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.036016 1138 sys_catalog.cc:455] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } } }
02:06:21.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.036643 1126 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } has no permanent_uuid. Determining permanent_uuid...
02:06:21.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.037312 1138 sys_catalog.cc:458] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [sys.catalog]: This master's current role is: FOLLOWER
02:06:21.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.040562 949 sys_catalog.cc:564] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [sys.catalog]: configured and running, proceeding with master startup.
02:06:21.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:21 dist-test-slave-szd5 krb5kdc[863](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778780, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.252@KRBTEST.COM for kudu/127.0.212.252@KRBTEST.COM
02:06:21.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:21.075596 903 tablet.cc:2378] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:06:21.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.081586 1126 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079: Bootstrap starting.
02:06:21.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.097604 1126 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079: Neither blocks nor log segments found. Creating new log.
02:06:21.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.106653 1126 log.cc:826] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079: Log is configured to *not* fsync() on all Append() calls
02:06:21.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.126596 1126 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079: No bootstrap required, opened a new log
02:06:21.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:21.151227 1150 catalog_manager.cc:1560] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:06:21.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:21.151710 1150 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:06:21.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.165741 1126 raft_consensus.cc:357] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } }
02:06:21.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.166925 1126 raft_consensus.cc:383] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:06:21.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.167320 1126 raft_consensus.cc:738] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: bdab70b6529844f5a3f4d6f008ba1079, State: Initialized, Role: FOLLOWER
02:06:21.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.168537 1126 consensus_queue.cc:260] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } }
02:06:21.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.174979 1152 sys_catalog.cc:455] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } } }
02:06:21.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.176376 1152 sys_catalog.cc:458] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [sys.catalog]: This master's current role is: FOLLOWER
02:06:21.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.177631 1126 sys_catalog.cc:564] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [sys.catalog]: configured and running, proceeding with master startup.
02:06:21.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.147342 880 master_runner.cc:186] Error getting master registration for 127.0.212.253:32829: OK, instance_id { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" instance_seqno: 1756778777645870 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:06:21.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.191515 880 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:06:21.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.249737 1152 raft_consensus.cc:491] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:06:21.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.250396 1152 raft_consensus.cc:513] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } }
02:06:21.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.255620 924 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "bdab70b6529844f5a3f4d6f008ba1079" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d068f217170340f9b4955443ddfda35f" is_pre_election: true
02:06:21.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.256685 924 raft_consensus.cc:2466] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate bdab70b6529844f5a3f4d6f008ba1079 in term 0.
02:06:21.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:21.257716 1163 catalog_manager.cc:1560] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:06:21.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.259204 1076 leader_election.cc:304] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: bdab70b6529844f5a3f4d6f008ba1079, d068f217170340f9b4955443ddfda35f; no voters:
02:06:21.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:21.258116 1163 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:06:21.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.268463 1152 leader_election.cc:290] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d068f217170340f9b4955443ddfda35f (127.0.212.254:46217), ced15ce4e97b4e92baa9efefcecd6a36 (127.0.212.253:32829)
02:06:21.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.269443 1152 raft_consensus.cc:2802] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 0 FOLLOWER]: Leader pre-election won for term 1
02:06:21.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.269990 1152 raft_consensus.cc:491] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:06:21.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.270552 1152 raft_consensus.cc:3058] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 0 FOLLOWER]: Advancing to term 1
02:06:21.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.284220 1152 raft_consensus.cc:513] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } }
02:06:21.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.293928 924 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "bdab70b6529844f5a3f4d6f008ba1079" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d068f217170340f9b4955443ddfda35f"
02:06:21.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.294602 924 raft_consensus.cc:3058] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [term 0 FOLLOWER]: Advancing to term 1
02:06:21.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.248287 1046 master_runner.cc:186] Error getting master registration for 127.0.212.253:32829: OK, instance_id { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" instance_seqno: 1756778777645870 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:06:21.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.300660 1046 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:06:21.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.306488 924 raft_consensus.cc:2466] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate bdab70b6529844f5a3f4d6f008ba1079 in term 1.
02:06:21.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.308979 1152 leader_election.cc:290] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [CANDIDATE]: Term 1 election: Requested vote from peers d068f217170340f9b4955443ddfda35f (127.0.212.254:46217), ced15ce4e97b4e92baa9efefcecd6a36 (127.0.212.253:32829)
02:06:21.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.310572 1076 leader_election.cc:304] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: bdab70b6529844f5a3f4d6f008ba1079, d068f217170340f9b4955443ddfda35f; no voters:
02:06:21.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.311844 1152 raft_consensus.cc:2802] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 1 FOLLOWER]: Leader election won for term 1
02:06:21.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.314754 1152 raft_consensus.cc:695] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 1 LEADER]: Becoming Leader. State: Replica: bdab70b6529844f5a3f4d6f008ba1079, State: Running, Role: LEADER
02:06:21.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.316298 1152 consensus_queue.cc:237] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } }
02:06:21.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.335139 1165 sys_catalog.cc:455] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [sys.catalog]: SysCatalogTable state changed. Reason: New leader bdab70b6529844f5a3f4d6f008ba1079. Latest consensus state: current_term: 1 leader_uuid: "bdab70b6529844f5a3f4d6f008ba1079" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } } }
02:06:21.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.336265 1165 sys_catalog.cc:458] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [sys.catalog]: This master's current role is: LEADER
02:06:21.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.347265 1167 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
02:06:21.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.360353 1167 catalog_manager.cc:1486] Initializing Kudu cluster ID...
02:06:21.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:21.399984 1080 tablet.cc:2378] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:06:21.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.406524 924 raft_consensus.cc:1273] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [term 1 FOLLOWER]: Refusing update from remote peer bdab70b6529844f5a3f4d6f008ba1079: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:06:21.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.408612 1152 consensus_queue.cc:1035] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
02:06:21.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.464303 1138 sys_catalog.cc:455] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [sys.catalog]: SysCatalogTable state changed. Reason: New leader bdab70b6529844f5a3f4d6f008ba1079. Latest consensus state: current_term: 1 leader_uuid: "bdab70b6529844f5a3f4d6f008ba1079" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } } }
02:06:21.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.465528 1138 sys_catalog.cc:458] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [sys.catalog]: This master's current role is: FOLLOWER
02:06:21.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.475414 1168 mvcc.cc:204] Tried to move back new op lower bound from 7195765888606420992 to 7195765888284553216. Current Snapshot: MvccSnapshot[applied={T|T < 7195765888606420992}]
02:06:21.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.496073 1152 sys_catalog.cc:455] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "bdab70b6529844f5a3f4d6f008ba1079" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } } }
02:06:21.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.497066 1152 sys_catalog.cc:458] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [sys.catalog]: This master's current role is: LEADER
02:06:21.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.499240 1167 catalog_manager.cc:1349] Generated new cluster ID: 0d51723196934732bbe6c094bcb5b19d
02:06:21.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.499596 1167 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
02:06:21.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.503938 1169 mvcc.cc:204] Tried to move back new op lower bound from 7195765888606420992 to 7195765888284553216. Current Snapshot: MvccSnapshot[applied={T|T < 7195765888606420992}]
02:06:21.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.520772 1138 sys_catalog.cc:455] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "bdab70b6529844f5a3f4d6f008ba1079" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } } }
02:06:21.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.521685 1138 sys_catalog.cc:458] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [sys.catalog]: This master's current role is: FOLLOWER
02:06:21.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.577989 1167 catalog_manager.cc:1372] Generated new certificate authority record
02:06:21.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.580097 1167 catalog_manager.cc:1506] Loading token signing keys...
02:06:21.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.618438 1167 catalog_manager.cc:5955] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079: Generated new TSK 0
02:06:21.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.619865 1167 catalog_manager.cc:1516] Initializing in-progress tserver states...
02:06:21.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.0.212.193@KRBTEST.COM; defaulting to no policy
02:06:21.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:06:21.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.0.212.193@KRBTEST.COM" created.
02:06:21.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:06:21.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.0.212.193 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab.
02:06:21.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.0.212.193 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab.
02:06:21.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.0.212.193@KRBTEST.COM; defaulting to no policy
02:06:21.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:06:21.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.0.212.193@KRBTEST.COM" created.
02:06:21.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:06:21.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.0.212.193 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab.
02:06:21.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.0.212.193 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab.
02:06:21.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:21.808182 851 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu
02:06:21.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu
02:06:21.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-0/wal
02:06:21.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-0/data
02:06:21.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:06:21.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:06:21.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:06:21.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:21.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:21.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:21.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-0/logs
02:06:21.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-0/data/info.pb
02:06:21.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:21.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:06:21.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:06:21.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:06:21.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:06:21.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:21.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:21.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:21.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:06:21.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:06:21.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.212.193:0
02:06:21.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.212.193
02:06:21.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.212.193
02:06:21.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:21.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.212.254:46217,127.0.212.253:32829,127.0.212.252:41741
02:06:21.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab
02:06:21.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.212.193
02:06:21.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:21.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:06:21.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:06:21.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:06:21.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.212.212:38213
02:06:21.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:21.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:21.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:21.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:06:22.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:22.158390 1150 catalog_manager.cc:1261] Loaded cluster ID: 0d51723196934732bbe6c094bcb5b19d
02:06:22.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:22.159008 1150 catalog_manager.cc:1554] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f: loading cluster ID for follower catalog manager: success
02:06:22.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:22.175416 1150 catalog_manager.cc:1576] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f: acquiring CA information for follower catalog manager: success
02:06:22.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:22.208129 1150 catalog_manager.cc:1604] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:06:22.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:22.260721 1036 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } attempt: 8
02:06:22.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:22 dist-test-slave-szd5 krb5kdc[863](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778777, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.253@KRBTEST.COM for kudu/127.0.212.252@KRBTEST.COM
02:06:22.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:22.440213 1036 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36: Bootstrap starting.
02:06:22.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:22.520642 1036 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36: Neither blocks nor log segments found. Creating new log.
02:06:22.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:22.524741 1036 log.cc:826] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36: Log is configured to *not* fsync() on all Append() calls
02:06:22.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:22.564273 1036 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36: No bootstrap required, opened a new log
02:06:22.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:22.605020 1036 raft_consensus.cc:357] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } }
02:06:22.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:22.606101 1036 raft_consensus.cc:383] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:06:22.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:22.606485 1036 raft_consensus.cc:738] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ced15ce4e97b4e92baa9efefcecd6a36, State: Initialized, Role: FOLLOWER
02:06:22.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:22.607988 1036 consensus_queue.cc:260] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } }
02:06:22.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:22.626480 1185 sys_catalog.cc:455] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } } }
02:06:22.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:22.627657 1185 sys_catalog.cc:458] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36 [sys.catalog]: This master's current role is: FOLLOWER
02:06:22.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:22.629479 1036 sys_catalog.cc:564] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36 [sys.catalog]: configured and running, proceeding with master startup.
02:06:22.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:22.630118 1008 raft_consensus.cc:3058] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36 [term 0 FOLLOWER]: Advancing to term 1
02:06:22.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:22.636967 1010 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "bdab70b6529844f5a3f4d6f008ba1079" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" is_pre_election: true
02:06:22.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:22.638348 1010 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:49324 (request call id 2) took 1369 ms (client timeout 1605 ms). Trace:
02:06:22.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:22.640206 1009 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "bdab70b6529844f5a3f4d6f008ba1079" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ced15ce4e97b4e92baa9efefcecd6a36"
02:06:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:22.641552 1009 rpcz_store.cc:275] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:49324 (request call id 3) took 1333 ms. Trace:
02:06:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:22.641990 1009 rpcz_store.cc:276] 0902 02:06:21.308741 (+ 0us) service_pool.cc:168] Inserting onto call queue
02:06:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 02:06:21.308984 (+ 243us) service_pool.cc:225] Handling call
02:06:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 02:06:22.641475 (+1332491us) inbound_call.cc:173] Queueing success response
02:06:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
02:06:22.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:22.638839 1010 rpcz_store.cc:269] 0902 02:06:21.268909 (+ 0us) service_pool.cc:168] Inserting onto call queue
02:06:22.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 02:06:21.269224 (+ 315us) service_pool.cc:225] Handling call
02:06:22.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 02:06:22.638261 (+1369037us) inbound_call.cc:173] Queueing success response
02:06:22.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
02:06:22.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:22.656620 1008 raft_consensus.cc:1273] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36 [term 1 FOLLOWER]: Refusing update from remote peer bdab70b6529844f5a3f4d6f008ba1079: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:06:22.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:22.657454 1008 rpcz_store.cc:275] Call kudu.consensus.ConsensusService.UpdateConsensus from 127.0.0.1:49324 (request call id 4) took 1251 ms. Trace:
02:06:22.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:22.657785 1008 rpcz_store.cc:276] 0902 02:06:21.406307 (+ 0us) service_pool.cc:168] Inserting onto call queue
02:06:22.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 02:06:21.406529 (+ 222us) service_pool.cc:225] Handling call
02:06:22.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 02:06:22.629804 (+1223275us) raft_consensus.cc:1469] Updating replica for 0 ops
02:06:22.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 02:06:22.657078 (+ 27274us) raft_consensus.cc:1695] Filling consensus response to leader.
02:06:22.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 02:06:22.657383 (+ 305us) inbound_call.cc:173] Queueing success response
02:06:22.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
02:06:22.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:22.672453 1190 consensus_queue.cc:1035] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:06:22.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:22.809882 1185 sys_catalog.cc:455] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36 [sys.catalog]: SysCatalogTable state changed. Reason: New leader bdab70b6529844f5a3f4d6f008ba1079. Latest consensus state: current_term: 1 leader_uuid: "bdab70b6529844f5a3f4d6f008ba1079" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } } }
02:06:22.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:22.810626 1185 sys_catalog.cc:458] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36 [sys.catalog]: This master's current role is: FOLLOWER
02:06:22.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:22.810295 1199 sys_catalog.cc:455] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "bdab70b6529844f5a3f4d6f008ba1079" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } } }
02:06:22.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:22.811344 1199 sys_catalog.cc:458] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36 [sys.catalog]: This master's current role is: FOLLOWER
02:06:22.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:22.823647 1177 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:06:22.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:22.824298 1177 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:06:22.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:22.825438 1177 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:06:22.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:22.896817 1205 catalog_manager.cc:1261] Loaded cluster ID: 0d51723196934732bbe6c094bcb5b19d
02:06:22.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:22.897125 1205 catalog_manager.cc:1554] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36: loading cluster ID for follower catalog manager: success
02:06:22.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:22.907622 1205 catalog_manager.cc:1576] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36: acquiring CA information for follower catalog manager: success
02:06:22.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:22.915215 1205 catalog_manager.cc:1604] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:06:22.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:22.966558 1177 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:06:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:22.969741 1177 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.212.193
02:06:23.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:23.038599 1177 tablet_server_runner.cc:78] Tablet server non-default flags:
02:06:23.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:23.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.212.212:38213
02:06:23.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:23.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:23.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-0/data
02:06:23.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-0/wal
02:06:23.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab
02:06:23.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.212.193
02:06:23.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:23.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:23.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.212.193:0
02:06:23.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:06:23.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:23.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:06:23.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:06:23.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:23.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:23.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-0/data/info.pb
02:06:23.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.212.193
02:06:23.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:23.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.212.254:46217,127.0.212.253:32829,127.0.212.252:41741
02:06:23.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:06:23.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:23.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:06:23.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:06:23.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:23.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.212.193
02:06:23.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-0/logs
02:06:23.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:23.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:06:23.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:06:23.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:06:23.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:06:23.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
02:06:23.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:06:23.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 01:43:19 UTC on 5fd53c4cbb9d
02:06:23.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7857
02:06:23.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:06:23.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:23.040479 1177 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:06:23.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:23.042016 1177 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:06:23.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:23.083827 1191 sys_catalog.cc:455] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "bdab70b6529844f5a3f4d6f008ba1079" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } } }
02:06:23.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:23.086643 1191 sys_catalog.cc:458] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [sys.catalog]: This master's current role is: LEADER
02:06:23.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:23.104249 1208 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:24.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:24.192425 1031 debug-util.cc:398] Leaking SignalData structure 0x7b0800071680 after lost signal to thread 961
02:06:24.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:24.193301 1031 debug-util.cc:398] Leaking SignalData structure 0x7b0800071560 after lost signal to thread 1034
02:06:24.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:24 dist-test-slave-szd5 krb5kdc[863](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778784, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:24.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:24.674772 893 init.cc:295] Successfully reacquired a new kerberos TGT
02:06:25.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:24.477758 1207 debug-util.cc:398] Leaking SignalData structure 0x7b0800037b60 after lost signal to thread 1177
02:06:25.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:25.153321 1207 kernel_stack_watchdog.cc:198] Thread 1177 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 364ms:
02:06:25.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:06:25.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:06:25.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:06:25.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:06:25.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:23.115231 1210 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:25.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:25.159786 1177 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.048s user 0.487s sys 0.887s
02:06:25.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:25.160149 1177 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.049s user 0.487s sys 0.887s
02:06:25.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:25.197855 1177 server_base.cc:1047] running on GCE node
02:06:25.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:25.198028 1215 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:25.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:25.199450 1177 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:06:25.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:25.216378 1177 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:06:25.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:25.232290 1177 hybrid_clock.cc:648] HybridClock initialized: now 1756778785232199 us; error 82 us; skew 500 ppm
02:06:25.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:25.233511 1177 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:06:25.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:25 dist-test-slave-szd5 krb5kdc[863](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778785, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.193@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:25.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:25.253014 1177 init.cc:377] Logged in from keytab as kudu/127.0.212.193@KRBTEST.COM (short username kudu)
02:06:25.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:25.282851 1177 webserver.cc:480] Webserver started at http://127.0.212.193:39859/ using document root <none> and password file <none>
02:06:25.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:25.284157 1177 fs_manager.cc:362] Metadata directory not provided
02:06:25.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:25.284484 1177 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:06:25.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:25.285084 1177 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
02:06:25.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:25.300344 1177 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-0/data/instance:
02:06:25.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ce46aaca229a4a5a9e62b9338cac9b42"
02:06:25.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 02:06:25 on dist-test-slave-szd5"
02:06:25.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:25.301995 1177 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-0/wal/instance:
02:06:25.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ce46aaca229a4a5a9e62b9338cac9b42"
02:06:25.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 02:06:25 on dist-test-slave-szd5"
02:06:25.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:25.332444 1177 fs_manager.cc:696] Time spent creating directory manager: real 0.030s user 0.009s sys 0.005s
02:06:25.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:25.354173 1221 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:06:25.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:25.355758 1177 fs_manager.cc:730] Time spent opening block manager: real 0.019s user 0.004s sys 0.003s
02:06:25.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:25.356165 1177 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-0/data,/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-0/wal
02:06:25.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ce46aaca229a4a5a9e62b9338cac9b42"
02:06:25.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 02:06:25 on dist-test-slave-szd5"
02:06:25.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:25.356629 1177 fs_report.cc:389] FS layout report
02:06:25.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:06:25.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-0/wal
02:06:25.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-0/wal
02:06:25.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-0/data/data
02:06:25.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:06:25.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:06:25.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:06:25.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:06:25.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:06:25.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:06:25.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:06:25.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:06:25.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:06:25.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:06:25.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:25.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:25.599718 1177 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:06:25.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:25.618582 1177 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:25.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:25.619426 1177 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:25.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:25.635739 1177 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:06:25.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:25.655591 1177 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:06:25.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:25.655880 1177 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.001s sys 0.000s
02:06:25.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:25.656157 1177 ts_tablet_manager.cc:610] Registered 0 tablets
02:06:25.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:25.656375 1177 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:06:26.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:26.503664 1177 rpc_server.cc:307] RPC server started. Bound to: 127.0.212.193:44097
02:06:26.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:26.503952 1337 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.212.193:44097 every 8 connection(s)
02:06:26.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:26.507486 1177 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-0/data/info.pb
02:06:26.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:26.515031 851 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu as pid 1177
02:06:26.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:26.515596 851 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-0/wal/instance
02:06:26.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.0.212.194@KRBTEST.COM; defaulting to no policy
02:06:26.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:06:26.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.0.212.194@KRBTEST.COM" created.
02:06:26.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:26 dist-test-slave-szd5 krb5kdc[863](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778785, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.193@KRBTEST.COM for kudu/127.0.212.254@KRBTEST.COM
02:06:26.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:26 dist-test-slave-szd5 krb5kdc[863](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778785, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.193@KRBTEST.COM for kudu/127.0.212.253@KRBTEST.COM
02:06:26.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:26 dist-test-slave-szd5 krb5kdc[863](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778785, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.193@KRBTEST.COM for kudu/127.0.212.252@KRBTEST.COM
02:06:26.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:26.643226 1338 heartbeater.cc:344] Connected to a master server at 127.0.212.252:41741
02:06:26.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:26.643880 1338 heartbeater.cc:461] Registering TS with master...
02:06:26.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:26.645912 1338 heartbeater.cc:507] Master 127.0.212.252:41741 requested a full tablet report, sending...
02:06:26.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:26.646950 1342 heartbeater.cc:344] Connected to a master server at 127.0.212.254:46217
02:06:26.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:26.647379 1342 heartbeater.cc:461] Registering TS with master...
02:06:26.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:26.648962 1342 heartbeater.cc:507] Master 127.0.212.254:46217 requested a full tablet report, sending...
02:06:26.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:26.652446 1091 ts_manager.cc:194] Registered new tserver with Master: ce46aaca229a4a5a9e62b9338cac9b42 (127.0.212.193:44097)
02:06:26.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:26.655001 1091 master_service.cc:496] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.0.212.193@KRBTEST.COM'} at 127.0.212.193:40311
02:06:26.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:06:26.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.0.212.194 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab.
02:06:26.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.0.212.194 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab.
02:06:26.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:26.658535 851 mini_kdc.cc:279] Time spent creating service keytab for kudu/127.0.212.194: real 0.129s user 0.000s sys 0.022s
02:06:26.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:26.661034 1339 heartbeater.cc:344] Connected to a master server at 127.0.212.253:32829
02:06:26.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:26.661473 1339 heartbeater.cc:461] Registering TS with master...
02:06:26.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:26.662129 914 ts_manager.cc:194] Registered new tserver with Master: ce46aaca229a4a5a9e62b9338cac9b42 (127.0.212.193:44097)
02:06:26.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:26.663082 1339 heartbeater.cc:507] Master 127.0.212.253:32829 requested a full tablet report, sending...
02:06:26.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:26.667605 1000 ts_manager.cc:194] Registered new tserver with Master: ce46aaca229a4a5a9e62b9338cac9b42 (127.0.212.193:44097)
02:06:26.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.0.212.194@KRBTEST.COM; defaulting to no policy
02:06:26.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:06:26.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.0.212.194@KRBTEST.COM" created.
02:06:26.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:06:26.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.0.212.194 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab.
02:06:26.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.0.212.194 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab.
02:06:26.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:26.740550 851 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu
02:06:26.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu
02:06:26.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-1/wal
02:06:26.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-1/data
02:06:26.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:06:26.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:06:26.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:06:26.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:26.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:26.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:26.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-1/logs
02:06:26.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-1/data/info.pb
02:06:26.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:26.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:06:26.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:06:26.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:06:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:06:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:06:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:06:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.212.194:0
02:06:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.212.194
02:06:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.212.194
02:06:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.212.254:46217,127.0.212.253:32829,127.0.212.252:41741
02:06:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab
02:06:26.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.212.194
02:06:26.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:26.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:06:26.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:06:26.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:06:26.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.212.212:38213
02:06:26.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:26.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:26.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:26.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:06:27.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:27 dist-test-slave-szd5 krb5kdc[863](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778787, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:27.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:27.430155 979 init.cc:295] Successfully reacquired a new kerberos TGT
02:06:27.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:27.680763 1338 heartbeater.cc:499] Master 127.0.212.252:41741 was elected leader, sending a full tablet report...
02:06:28.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:28.334707 305 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.31.211.126:43055, user_credentials={real_user=slave}} blocked reactor thread for 83386.3us
02:06:28.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:28.316627 1078 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.212.253:32829, user_credentials={real_user=slave}} blocked reactor thread for 103730us
02:06:28.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:28.965837 1352 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:06:28.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:28.966521 1352 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:06:28.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:28.967732 1352 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:06:29.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:29.044731 1352 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:06:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:29.046567 1352 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.212.194
02:06:29.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:29.119302 1352 tablet_server_runner.cc:78] Tablet server non-default flags:
02:06:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.212.212:38213
02:06:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-1/data
02:06:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-1/wal
02:06:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab
02:06:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.212.194
02:06:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.212.194:0
02:06:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:06:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:06:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:06:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-1/data/info.pb
02:06:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.212.194
02:06:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.212.254:46217,127.0.212.253:32829,127.0.212.252:41741
02:06:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:06:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:06:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:06:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.212.194
02:06:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-1/logs
02:06:29.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:29.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:06:29.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:06:29.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:06:29.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:06:29.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
02:06:29.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:06:29.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 01:43:19 UTC on 5fd53c4cbb9d
02:06:29.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7857
02:06:29.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:06:29.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:29.121052 1352 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:06:29.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:29.122581 1352 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:06:29.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:29.163987 1366 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:30.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:30 dist-test-slave-szd5 krb5kdc[863](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778790, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:30.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:30.434881 1069 init.cc:295] Successfully reacquired a new kerberos TGT
02:06:30.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:30.558915 1365 debug-util.cc:398] Leaking SignalData structure 0x7b0800037b60 after lost signal to thread 1352
02:06:30.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:30.803414 1352 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.641s user 0.572s sys 0.932s
02:06:30.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:29.165452 1367 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:30.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:30.803839 1352 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.642s user 0.573s sys 0.932s
02:06:30.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:30.815323 1373 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:30.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:30.815937 1352 server_base.cc:1047] running on GCE node
02:06:30.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:30.817565 1352 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:06:30.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:30.820905 1352 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:06:30.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:30.822299 1352 hybrid_clock.cc:648] HybridClock initialized: now 1756778790822264 us; error 36 us; skew 500 ppm
02:06:30.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:30.823223 1352 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:06:30.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:30 dist-test-slave-szd5 krb5kdc[863](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778790, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.194@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:30.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:30.831485 1352 init.cc:377] Logged in from keytab as kudu/127.0.212.194@KRBTEST.COM (short username kudu)
02:06:30.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:30.839648 1352 webserver.cc:480] Webserver started at http://127.0.212.194:42665/ using document root <none> and password file <none>
02:06:30.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:30.840728 1352 fs_manager.cc:362] Metadata directory not provided
02:06:30.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:30.840963 1352 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:06:30.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:30.841470 1352 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
02:06:30.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:30.847463 1352 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-1/data/instance:
02:06:30.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4ac25f02a47d4bc2a47eb802eb602ee9"
02:06:30.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 02:06:30 on dist-test-slave-szd5"
02:06:30.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:30.849056 1352 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-1/wal/instance:
02:06:30.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4ac25f02a47d4bc2a47eb802eb602ee9"
02:06:30.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 02:06:30 on dist-test-slave-szd5"
02:06:30.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:30.862125 1352 fs_manager.cc:696] Time spent creating directory manager: real 0.012s user 0.012s sys 0.000s
02:06:30.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:30.873001 1380 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:06:30.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:30.874415 1352 fs_manager.cc:730] Time spent opening block manager: real 0.007s user 0.003s sys 0.005s
02:06:30.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:30.874714 1352 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-1/data,/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-1/wal
02:06:30.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4ac25f02a47d4bc2a47eb802eb602ee9"
02:06:30.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 02:06:30 on dist-test-slave-szd5"
02:06:30.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:30.875031 1352 fs_report.cc:389] FS layout report
02:06:30.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:06:30.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-1/wal
02:06:30.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-1/wal
02:06:30.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-1/data/data
02:06:30.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:06:30.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:06:30.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:06:30.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:06:30.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:06:30.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:06:30.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:06:30.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:06:30.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:06:30.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:06:30.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:30.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:30.937883 1352 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:06:30.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:30.968236 1352 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:30.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:30.968848 1352 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:30.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:30.974296 1352 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:06:30.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:30.982221 1352 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:06:30.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:30.982558 1352 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:06:30.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:30.982853 1352 ts_tablet_manager.cc:610] Registered 0 tablets
02:06:30.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:30.983081 1352 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:06:31.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:31.333580 1352 rpc_server.cc:307] RPC server started. Bound to: 127.0.212.194:39417
02:06:31.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:31.333873 1493 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.212.194:39417 every 8 connection(s)
02:06:31.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:31.336310 1352 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-1/data/info.pb
02:06:31.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:31.347280 851 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu as pid 1352
02:06:31.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:31.348134 851 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-1/wal/instance
02:06:31.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.0.212.195@KRBTEST.COM; defaulting to no policy
02:06:31.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:06:31.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.0.212.195@KRBTEST.COM" created.
02:06:31.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:31 dist-test-slave-szd5 krb5kdc[863](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778790, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.194@KRBTEST.COM for kudu/127.0.212.253@KRBTEST.COM
02:06:31.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:31 dist-test-slave-szd5 krb5kdc[863](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778790, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.194@KRBTEST.COM for kudu/127.0.212.254@KRBTEST.COM
02:06:31.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:31 dist-test-slave-szd5 krb5kdc[863](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778790, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.194@KRBTEST.COM for kudu/127.0.212.252@KRBTEST.COM
02:06:31.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:31.441247 1495 heartbeater.cc:344] Connected to a master server at 127.0.212.253:32829
02:06:31.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:31.441903 1495 heartbeater.cc:461] Registering TS with master...
02:06:31.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:31.443820 1495 heartbeater.cc:507] Master 127.0.212.253:32829 requested a full tablet report, sending...
02:06:31.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:31.447001 1500 heartbeater.cc:344] Connected to a master server at 127.0.212.254:46217
02:06:31.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:31.447398 1500 heartbeater.cc:461] Registering TS with master...
02:06:31.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:31.448860 1500 heartbeater.cc:507] Master 127.0.212.254:46217 requested a full tablet report, sending...
02:06:31.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:31.452970 1000 ts_manager.cc:194] Registered new tserver with Master: 4ac25f02a47d4bc2a47eb802eb602ee9 (127.0.212.194:39417)
02:06:31.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:31.453862 914 ts_manager.cc:194] Registered new tserver with Master: 4ac25f02a47d4bc2a47eb802eb602ee9 (127.0.212.194:39417)
02:06:31.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:31.455642 1494 heartbeater.cc:344] Connected to a master server at 127.0.212.252:41741
02:06:31.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:31.455963 1494 heartbeater.cc:461] Registering TS with master...
02:06:31.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:31.456895 1494 heartbeater.cc:507] Master 127.0.212.252:41741 requested a full tablet report, sending...
02:06:31.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:31.459630 1091 ts_manager.cc:194] Registered new tserver with Master: 4ac25f02a47d4bc2a47eb802eb602ee9 (127.0.212.194:39417)
02:06:31.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:06:31.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.0.212.195 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab.
02:06:31.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.0.212.195 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab.
02:06:31.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:31.461766 1091 master_service.cc:496] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.0.212.194@KRBTEST.COM'} at 127.0.212.194:48341
02:06:31.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.0.212.195@KRBTEST.COM; defaulting to no policy
02:06:31.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:06:31.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.0.212.195@KRBTEST.COM" created.
02:06:31.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:06:31.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.0.212.195 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab.
02:06:31.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.0.212.195 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab.
02:06:31.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:31.508931 851 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu
02:06:31.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu
02:06:31.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-2/wal
02:06:31.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-2/data
02:06:31.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:06:31.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:06:31.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:06:31.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:31.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:31.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:31.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-2/logs
02:06:31.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-2/data/info.pb
02:06:31.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:31.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:06:31.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:06:31.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:06:31.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:06:31.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:31.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:31.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:31.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:06:31.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:06:31.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.212.195:0
02:06:31.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.212.195
02:06:31.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.212.195
02:06:31.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:31.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.212.254:46217,127.0.212.253:32829,127.0.212.252:41741
02:06:31.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab
02:06:31.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.212.195
02:06:31.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:31.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:06:31.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:06:31.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:06:31.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.212.212:38213
02:06:31.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:31.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:31.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:31.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:06:31.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:31.931481 1508 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:06:31.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:31.932119 1508 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:06:31.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:31.932848 1508 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:06:31.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:31.965718 1508 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:06:31.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:31.966789 1508 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.212.195
02:06:32.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:32.004197 1508 tablet_server_runner.cc:78] Tablet server non-default flags:
02:06:32.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:32.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.212.212:38213
02:06:32.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:32.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:32.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-2/data
02:06:32.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-2/wal
02:06:32.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab
02:06:32.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.212.195
02:06:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.212.195:0
02:06:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:06:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:06:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:06:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-2/data/info.pb
02:06:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.212.195
02:06:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.212.254:46217,127.0.212.253:32829,127.0.212.252:41741
02:06:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:06:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:32.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:06:32.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:06:32.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:32.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.212.195
02:06:32.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-2/logs
02:06:32.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:32.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:06:32.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:06:32.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:06:32.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:06:32.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
02:06:32.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:06:32.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 01:43:19 UTC on 5fd53c4cbb9d
02:06:32.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7857
02:06:32.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:06:32.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:32.005817 1508 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:06:32.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:32.007083 1508 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:06:32.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:32.025300 1514 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:32.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:32.465973 1494 heartbeater.cc:499] Master 127.0.212.252:41741 was elected leader, sending a full tablet report...
02:06:34.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:34.031903 305 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.31.211.126:43055, user_credentials={real_user=slave}} blocked reactor thread for 332723us
02:06:34.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:33.437795 1513 debug-util.cc:398] Leaking SignalData structure 0x7b0800037b60 after lost signal to thread 1508
02:06:34.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:34.457826 1508 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.431s user 0.000s sys 0.002s
02:06:34.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:34.460237 1508 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.434s user 0.000s sys 0.003s
02:06:34.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:34.461428 1516 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2432 milliseconds
02:06:34.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:32.025938 1515 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:34.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:34.462287 1508 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:06:34.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:34.463394 1517 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:34.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:34.467573 1508 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:06:34.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:34.471369 1508 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:06:34.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:34.472877 1508 hybrid_clock.cc:648] HybridClock initialized: now 1756778794472795 us; error 89 us; skew 500 ppm
02:06:34.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:34.473958 1508 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:06:34.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:34 dist-test-slave-szd5 krb5kdc[863](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778794, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.195@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:34.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:34.482919 1508 init.cc:377] Logged in from keytab as kudu/127.0.212.195@KRBTEST.COM (short username kudu)
02:06:34.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:34.493032 1508 webserver.cc:480] Webserver started at http://127.0.212.195:38733/ using document root <none> and password file <none>
02:06:34.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:34.494297 1508 fs_manager.cc:362] Metadata directory not provided
02:06:34.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:34.494593 1508 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:06:34.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:34.495277 1508 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
02:06:34.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:34.502883 1508 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-2/data/instance:
02:06:34.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8f26756046ee4fb0a2e55f50a20e45ab"
02:06:34.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 02:06:34 on dist-test-slave-szd5"
02:06:34.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:34.504618 1508 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-2/wal/instance:
02:06:34.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8f26756046ee4fb0a2e55f50a20e45ab"
02:06:34.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 02:06:34 on dist-test-slave-szd5"
02:06:34.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:34.517192 1508 fs_manager.cc:696] Time spent creating directory manager: real 0.012s user 0.010s sys 0.001s
02:06:34.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:34.525844 1530 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:06:34.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:34.527140 1508 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.005s sys 0.000s
02:06:34.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:34.527454 1508 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-2/data,/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-2/wal
02:06:34.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8f26756046ee4fb0a2e55f50a20e45ab"
02:06:34.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 02:06:34 on dist-test-slave-szd5"
02:06:34.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:34.527868 1508 fs_report.cc:389] FS layout report
02:06:34.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:06:34.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-2/wal
02:06:34.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-2/wal
02:06:34.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-2/data/data
02:06:34.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:06:34.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:06:34.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:06:34.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:06:34.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:06:34.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:06:34.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:06:34.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:06:34.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:06:34.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:06:34.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:34.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:34.601153 1508 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:06:34.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:34.607806 1508 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:34.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:34.608382 1508 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:34.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:34.612670 1508 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:06:34.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:34.620301 1508 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:06:34.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:34.620544 1508 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:06:34.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:34.620872 1508 ts_tablet_manager.cc:610] Registered 0 tablets
02:06:34.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:34.621042 1508 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:06:34.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:34 dist-test-slave-szd5 krb5kdc[863](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778794, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:34.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:34.680445 893 init.cc:295] Successfully reacquired a new kerberos TGT
02:06:34.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:34.978950 1508 rpc_server.cc:307] RPC server started. Bound to: 127.0.212.195:38071
02:06:34.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:34.979254 1642 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.212.195:38071 every 8 connection(s)
02:06:34.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:34.983647 1508 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-2/data/info.pb
02:06:34.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:34.988181 851 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu as pid 1508
02:06:34.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:34.988716 851 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/ts-2/wal/instance
02:06:35.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:35 dist-test-slave-szd5 krb5kdc[863](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778794, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.195@KRBTEST.COM for kudu/127.0.212.252@KRBTEST.COM
02:06:35.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:35 dist-test-slave-szd5 krb5kdc[863](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778794, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.195@KRBTEST.COM for kudu/127.0.212.254@KRBTEST.COM
02:06:35.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:35 dist-test-slave-szd5 krb5kdc[863](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778794, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.195@KRBTEST.COM for kudu/127.0.212.253@KRBTEST.COM
02:06:35.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:35.090454 1643 heartbeater.cc:344] Connected to a master server at 127.0.212.252:41741
02:06:35.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:35.090971 1643 heartbeater.cc:461] Registering TS with master...
02:06:35.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:35.092325 1643 heartbeater.cc:507] Master 127.0.212.252:41741 requested a full tablet report, sending...
02:06:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:35.095118 1091 ts_manager.cc:194] Registered new tserver with Master: 8f26756046ee4fb0a2e55f50a20e45ab (127.0.212.195:38071)
02:06:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:35.096853 1644 heartbeater.cc:344] Connected to a master server at 127.0.212.253:32829
02:06:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:35.097066 1091 master_service.cc:496] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.0.212.195@KRBTEST.COM'} at 127.0.212.195:34131
02:06:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:35.097167 1644 heartbeater.cc:461] Registering TS with master...
02:06:35.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:35.098078 1644 heartbeater.cc:507] Master 127.0.212.253:32829 requested a full tablet report, sending...
02:06:35.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:35.101018 1000 ts_manager.cc:194] Registered new tserver with Master: 8f26756046ee4fb0a2e55f50a20e45ab (127.0.212.195:38071)
02:06:35.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:35.128170 1647 heartbeater.cc:344] Connected to a master server at 127.0.212.254:46217
02:06:35.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:35.128769 1647 heartbeater.cc:461] Registering TS with master...
02:06:35.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:35.129833 1647 heartbeater.cc:507] Master 127.0.212.254:46217 requested a full tablet report, sending...
02:06:35.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:35.133656 914 ts_manager.cc:194] Registered new tserver with Master: 8f26756046ee4fb0a2e55f50a20e45ab (127.0.212.195:38071)
02:06:35.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:35.146306 851 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
02:06:35.164 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:06:35.165 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
02:06:35.174 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
02:06:35.175 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
02:06:35.178 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.0.212.254"
port: 46217
}
bound_http_address {
host: "127.0.212.254"
port: 44809
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.0.212.253"
port: 32829
}
bound_http_address {
host: "127.0.212.253"
port: 39491
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.0.212.252"
port: 41741
}
bound_http_address {
host: "127.0.212.252"
port: 33899
}
}
}
02:06:35.178 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
02:06:35.179 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.0.212.193"
port: 44097
}
bound_http_address {
host: "127.0.212.193"
port: 39859
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.0.212.194"
port: 39417
}
bound_http_address {
host: "127.0.212.194"
port: 42665
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.0.212.195"
port: 38071
}
bound_http_address {
host: "127.0.212.195"
port: 38733
}
}
}
02:06:35.180 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
02:06:35.181 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
02:06:35.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:35 dist-test-slave-szd5 krb5kdc[863](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778795, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:35.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
02:06:35.219 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:06:35.221 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/krb5cc
02:06:35.226 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:06:35.227 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/krb5cc
02:06:35.228 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:06:35.231 [DEBUG - Time-limited test] (SecurityContext.java:158) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
02:06:35.232 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
02:06:35.233 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
02:06:35.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:35 dist-test-slave-szd5 krb5kdc[863](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778795, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:35.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
02:06:35.263 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:06:35.264 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/krb5cc
02:06:35.265 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:06:35.266 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.0.212.252:41741
02:06:35.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:35 dist-test-slave-szd5 krb5kdc[863](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778795, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.193@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:35.267 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
02:06:35.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:35.267915 1217 init.cc:295] Successfully reacquired a new kerberos TGT
02:06:35.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:35.268024 851 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu with pid 1046
02:06:35.293 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:06:35.293 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.0.212.254:46217
02:06:35.293 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
02:06:35.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:35.293944 851 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu with pid 880
02:06:35.320 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:06:35.320 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.0.212.253:32829
02:06:35.321 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
02:06:35.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:35.321311 851 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu with pid 958
02:06:35.361 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:06:35.362 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.0.212.252:41741
02:06:35.365 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
02:06:35.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:35.368283 851 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu
02:06:35.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu
02:06:35.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/wal
02:06:35.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/data
02:06:35.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:06:35.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:06:35.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:06:35.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:35.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:35.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:35.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/logs
02:06:35.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/data/info.pb
02:06:35.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:35.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:06:35.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:06:35.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:06:35.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:06:35.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:35.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:35.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:35.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:06:35.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:06:35.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:35.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:35.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.212.252:41741
02:06:35.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.212.252
02:06:35.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33899
02:06:35.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab
02:06:35.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.212.252
02:06:35.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:35.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:06:35.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:06:35.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:06:35.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.212.212:38213
02:06:35.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:35.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:35.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:35.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:35.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.212.254:46217,127.0.212.253:32829,127.0.212.252:41741
02:06:35.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:35.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38193/jwks.json
02:06:35.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:06:35.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:06:35.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:35.506444 1494 heartbeater.cc:646] Failed to heartbeat to 127.0.212.252:41741 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.0.212.252:41741: connect: Connection refused (error 111)
02:06:35.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:35.688835 1658 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:06:35.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:35.689364 1658 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:06:35.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:35.690125 1658 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:06:35.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:35.721058 1658 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:06:35.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:35.721280 1658 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:06:35.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:35.721554 1658 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:06:35.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:35.721750 1658 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:06:35.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:35.721935 1658 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:06:35.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:35.722114 1658 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:06:35.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:35.722275 1658 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38193/jwks.json
02:06:35.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:35.758688 1658 master_runner.cc:386] Master server non-default flags:
02:06:35.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:35.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.212.212:38213
02:06:35.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:35.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:35.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/data
02:06:35.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/wal
02:06:35.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:06:35.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:35.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.212.254:46217,127.0.212.253:32829,127.0.212.252:41741
02:06:35.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab
02:06:35.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.212.252
02:06:35.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:35.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:35.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:35.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.212.252:41741
02:06:35.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:35.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:06:35.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:35.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38193/jwks.json
02:06:35.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:35.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:06:35.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:06:35.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:35.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:35.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/data/info.pb
02:06:35.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.212.252
02:06:35.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33899
02:06:35.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:06:35.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:35.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:06:35.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:06:35.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:35.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/logs
02:06:35.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:35.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:06:35.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:06:35.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:06:35.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:06:35.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
02:06:35.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:06:35.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 01:43:19 UTC on 5fd53c4cbb9d
02:06:35.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7857
02:06:35.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:06:35.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:35.760010 1658 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:06:35.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:35.761082 1658 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:06:35.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:35.772294 1668 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:35.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:35.983819 1339 heartbeater.cc:646] Failed to heartbeat to 127.0.212.253:32829 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.0.212.253:32829: connect: Connection refused (error 111)
02:06:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:36.106837 1643 heartbeater.cc:499] Master 127.0.212.252:41741 was elected leader, sending a full tablet report...
02:06:36.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:36.128880 1644 heartbeater.cc:646] Failed to heartbeat to 127.0.212.253:32829 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.0.212.253:32829: connect: Connection refused (error 111)
02:06:37.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:35.773869 1669 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:37.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:37.179749 1667 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 1658
02:06:37.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:37.453498 1658 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.679s user 0.547s sys 1.100s
02:06:37.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:37.453862 1658 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.679s user 0.547s sys 1.100s
02:06:37.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:37.458992 1670 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1683 milliseconds
02:06:37.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:37.460083 1671 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:37.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:37.460129 1658 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:06:37.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:37.461313 1658 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:06:37.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:37.463989 1658 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:06:37.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:37.465354 1658 hybrid_clock.cc:648] HybridClock initialized: now 1756778797465322 us; error 32 us; skew 500 ppm
02:06:37.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:37.466140 1658 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:06:37.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:37 dist-test-slave-szd5 krb5kdc[863](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778797, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:37.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:37.474395 1658 init.cc:377] Logged in from keytab as kudu/127.0.212.252@KRBTEST.COM (short username kudu)
02:06:37.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:37.480454 1658 webserver.cc:480] Webserver started at http://127.0.212.252:33899/ using document root <none> and password file <none>
02:06:37.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:37.481501 1658 fs_manager.cc:362] Metadata directory not provided
02:06:37.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:37.481699 1658 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:06:37.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:37.489545 1658 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.000s sys 0.004s
02:06:37.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:37.493990 1688 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:06:37.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:37.494941 1658 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
02:06:37.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:37.495258 1658 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/data,/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/wal
02:06:37.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bdab70b6529844f5a3f4d6f008ba1079"
02:06:37.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 02:06:20 on dist-test-slave-szd5"
02:06:37.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:37.497243 1658 fs_report.cc:389] FS layout report
02:06:37.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:06:37.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/wal
02:06:37.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/wal
02:06:37.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/data/data
02:06:37.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:06:37.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:06:37.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:06:37.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:06:37.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:06:37.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:06:37.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:06:37.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:06:37.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:06:37.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:06:37.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:37.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:37.621651 1658 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:06:37.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:37.647195 1658 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:37.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:37.649586 1658 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:37.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:37.874738 1658 rpc_server.cc:307] RPC server started. Bound to: 127.0.212.252:41741
02:06:37.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:37.875125 1740 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.212.252:41741 every 8 connection(s)
02:06:37.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:37.880852 1658 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/data/info.pb
02:06:37.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:37.887212 851 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu as pid 1658
02:06:37.888 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:06:37.888 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.0.212.254:46217
02:06:37.889 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
02:06:37.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:37.890699 851 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu
02:06:37.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu
02:06:37.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/wal
02:06:37.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/data
02:06:37.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:06:37.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:06:37.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:06:37.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:37.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:37.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:37.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/logs
02:06:37.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/data/info.pb
02:06:37.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:37.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:06:37.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:06:37.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:06:37.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:06:37.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:37.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:37.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:37.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:06:37.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:06:37.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:37.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:37.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.212.254:46217
02:06:37.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.212.254
02:06:37.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44809
02:06:37.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab
02:06:37.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.212.254
02:06:37.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:37.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:06:37.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:06:37.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:06:37.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.212.212:38213
02:06:37.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:37.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:37.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:37.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:37.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.212.254:46217,127.0.212.253:32829,127.0.212.252:41741
02:06:37.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:37.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38193/jwks.json
02:06:37.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:06:37.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:06:37.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:37.910212 1741 sys_catalog.cc:263] Verifying existing consensus state
02:06:37.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:37.928073 1741 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079: Bootstrap starting.
02:06:38.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:38.050515 1741 log.cc:826] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079: Log is configured to *not* fsync() on all Append() calls
02:06:38.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:38.078665 1741 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:06:38.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:38.079746 1741 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079: Bootstrap complete.
02:06:38.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:38 dist-test-slave-szd5 krb5kdc[863](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778795, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.193@KRBTEST.COM for kudu/127.0.212.252@KRBTEST.COM
02:06:38.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:38.109402 1338 heartbeater.cc:344] Connected to a master server at 127.0.212.252:41741
02:06:38.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:38.106899 1741 raft_consensus.cc:357] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } }
02:06:38.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:38.110018 1741 raft_consensus.cc:738] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: bdab70b6529844f5a3f4d6f008ba1079, State: Initialized, Role: FOLLOWER
02:06:38.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:38.110760 1741 consensus_queue.cc:260] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } }
02:06:38.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:38.114360 1748 sys_catalog.cc:455] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } } }
02:06:38.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:38.115284 1748 sys_catalog.cc:458] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [sys.catalog]: This master's current role is: FOLLOWER
02:06:38.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:38.116523 1741 sys_catalog.cc:564] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [sys.catalog]: configured and running, proceeding with master startup.
02:06:38.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:38.191955 1643 heartbeater.cc:344] Connected to a master server at 127.0.212.252:41741
02:06:38.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:38.192414 1643 heartbeater.cc:499] Master 127.0.212.252:41741 was elected leader, sending a full tablet report...
02:06:38.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:38.217043 1762 catalog_manager.cc:1261] Loaded cluster ID: 0d51723196934732bbe6c094bcb5b19d
02:06:38.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:38.217331 1762 catalog_manager.cc:1554] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079: loading cluster ID for follower catalog manager: success
02:06:38.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:38.225234 1762 catalog_manager.cc:1576] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079: acquiring CA information for follower catalog manager: success
02:06:38.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:38.231630 1762 catalog_manager.cc:1604] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:06:38.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:38.212523 1658 master_runner.cc:186] Error getting master registration for 127.0.212.254:46217: Network error: Client connection negotiation failed: client connection to 127.0.212.254:46217: connect: Connection refused (error 111),
02:06:38.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:38.241014 1658 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:06:38.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:38.409832 1743 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:06:38.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:38.410459 1743 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:06:38.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:38.411062 1743 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:06:38.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:38.445588 1743 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:06:38.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:38.445856 1743 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:06:38.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:38.446246 1743 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:06:38.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:38.446547 1743 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:06:38.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:38.446820 1743 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:06:38.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:38.447103 1743 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:06:38.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:38.447316 1743 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38193/jwks.json
02:06:38.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:38.487473 1743 master_runner.cc:386] Master server non-default flags:
02:06:38.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:38.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.212.212:38213
02:06:38.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:38.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:38.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/data
02:06:38.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/wal
02:06:38.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:06:38.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:38.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.212.254:46217,127.0.212.253:32829,127.0.212.252:41741
02:06:38.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab
02:06:38.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.212.254
02:06:38.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:38.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:38.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:38.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.212.254:46217
02:06:38.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:38.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:06:38.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:38.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38193/jwks.json
02:06:38.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:38.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:06:38.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:06:38.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:38.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:38.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/data/info.pb
02:06:38.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.212.254
02:06:38.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44809
02:06:38.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:06:38.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:38.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:06:38.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:06:38.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:38.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/logs
02:06:38.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:38.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:06:38.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:06:38.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:06:38.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:06:38.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
02:06:38.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:06:38.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 01:43:19 UTC on 5fd53c4cbb9d
02:06:38.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7857
02:06:38.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:06:38.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:38.489204 1743 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:06:38.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:38.490556 1743 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:06:38.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:38.536707 1773 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:38.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:38.548394 1494 heartbeater.cc:344] Connected to a master server at 127.0.212.252:41741
02:06:38.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:38.557616 1706 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "4ac25f02a47d4bc2a47eb802eb602ee9" instance_seqno: 1756778791257506) as {username='kudu', principal='kudu/127.0.212.194@KRBTEST.COM'} at 127.0.212.194:53103; Asking this server to re-register.
02:06:38.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:38.559289 1494 heartbeater.cc:461] Registering TS with master...
02:06:38.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:38.563196 1706 ts_manager.cc:194] Registered new tserver with Master: 4ac25f02a47d4bc2a47eb802eb602ee9 (127.0.212.194:39417)
02:06:39.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:39.116953 1706 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "ce46aaca229a4a5a9e62b9338cac9b42" instance_seqno: 1756778786352500) as {username='kudu', principal='kudu/127.0.212.193@KRBTEST.COM'} at 127.0.212.193:58047; Asking this server to re-register.
02:06:39.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:39.119427 1338 heartbeater.cc:461] Registering TS with master...
02:06:39.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:39.123487 1706 ts_manager.cc:194] Registered new tserver with Master: ce46aaca229a4a5a9e62b9338cac9b42 (127.0.212.193:44097)
02:06:39.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:39.195122 1643 heartbeater.cc:499] Master 127.0.212.252:41741 was elected leader, sending a full tablet report...
02:06:39.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:39.199944 1706 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "8f26756046ee4fb0a2e55f50a20e45ab" instance_seqno: 1756778794891758) as {username='kudu', principal='kudu/127.0.212.195@KRBTEST.COM'} at 127.0.212.195:35931; Asking this server to re-register.
02:06:39.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:39.202678 1643 heartbeater.cc:461] Registering TS with master...
02:06:39.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:39.207772 1706 ts_manager.cc:194] Registered new tserver with Master: 8f26756046ee4fb0a2e55f50a20e45ab (127.0.212.195:38071)
02:06:39.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:39.362318 1781 raft_consensus.cc:491] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:06:39.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:39.368274 1781 raft_consensus.cc:513] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } }
02:06:39.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:39.403980 1781 leader_election.cc:290] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers d068f217170340f9b4955443ddfda35f (127.0.212.254:46217), ced15ce4e97b4e92baa9efefcecd6a36 (127.0.212.253:32829)
02:06:39.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:39.465993 305 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.31.211.126:43055, user_credentials={real_user=slave}} blocked reactor thread for 54879.7us
02:06:39.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:39.475210 1693 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.212.253:32829: connect: Connection refused (error 111)
02:06:39.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:39.504278 1691 leader_election.cc:336] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer d068f217170340f9b4955443ddfda35f (127.0.212.254:46217): Network error: Client connection negotiation failed: client connection to 127.0.212.254:46217: connect: Connection refused (error 111)
02:06:39.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:39.514786 1693 leader_election.cc:336] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer ced15ce4e97b4e92baa9efefcecd6a36 (127.0.212.253:32829): Network error: Client connection negotiation failed: client connection to 127.0.212.253:32829: connect: Connection refused (error 111)
02:06:39.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:39.516112 1693 leader_election.cc:304] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: bdab70b6529844f5a3f4d6f008ba1079; no voters: ced15ce4e97b4e92baa9efefcecd6a36, d068f217170340f9b4955443ddfda35f
02:06:39.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:39.546655 1781 raft_consensus.cc:2747] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
02:06:40.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:39.929814 1770 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 1743
02:06:40.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:38.543401 1774 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:40.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:40.206627 1743 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.667s user 0.529s sys 0.998s
02:06:40.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:40.207032 1743 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.668s user 0.529s sys 0.998s
02:06:40.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:40.218531 1794 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:40.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:40.219983 1743 server_base.cc:1047] running on GCE node
02:06:40.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:40.221167 1743 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:06:40.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:40.224627 1743 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:06:40.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:40.226096 1743 hybrid_clock.cc:648] HybridClock initialized: now 1756778800226026 us; error 65 us; skew 500 ppm
02:06:40.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:40.226917 1743 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:06:40.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:40 dist-test-slave-szd5 krb5kdc[863](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778800, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:40.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:40.234130 1743 init.cc:377] Logged in from keytab as kudu/127.0.212.254@KRBTEST.COM (short username kudu)
02:06:40.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:40.241869 1743 webserver.cc:480] Webserver started at http://127.0.212.254:44809/ using document root <none> and password file <none>
02:06:40.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:40.242844 1743 fs_manager.cc:362] Metadata directory not provided
02:06:40.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:40.243081 1743 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:06:40.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:40.252715 1743 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.006s sys 0.002s
02:06:40.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:40.258342 1801 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:06:40.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:40.259681 1743 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.005s sys 0.001s
02:06:40.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:40.260085 1743 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/data,/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/wal
02:06:40.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d068f217170340f9b4955443ddfda35f"
02:06:40.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 02:06:14 on dist-test-slave-szd5"
02:06:40.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:40.262280 1743 fs_report.cc:389] FS layout report
02:06:40.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:06:40.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/wal
02:06:40.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/wal
02:06:40.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/data/data
02:06:40.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:06:40.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:06:40.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:06:40.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:06:40.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:06:40.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:06:40.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:06:40.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:06:40.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:06:40.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:06:40.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:40.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:40.324735 1743 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:06:40.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:40.331373 1743 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:40.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:40.331949 1743 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:40.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:40.461911 1743 rpc_server.cc:307] RPC server started. Bound to: 127.0.212.254:46217
02:06:40.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:40.461969 1852 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.212.254:46217 every 8 connection(s)
02:06:40.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:40.464895 1743 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/data/info.pb
02:06:40.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:40.473446 851 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu as pid 1743
02:06:40.474 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:06:40.475 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.0.212.253:32829
02:06:40.475 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
02:06:40.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:40.476966 851 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu
02:06:40.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu
02:06:40.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/wal
02:06:40.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/data
02:06:40.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:06:40.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:06:40.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:06:40.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:40.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:40.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:40.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/logs
02:06:40.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/data/info.pb
02:06:40.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:40.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:06:40.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:06:40.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:06:40.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:06:40.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:40.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:40.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:40.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:06:40.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:06:40.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:40.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:40.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.212.253:32829
02:06:40.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.212.253
02:06:40.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39491
02:06:40.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab
02:06:40.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.212.253
02:06:40.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:40.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:06:40.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:06:40.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:06:40.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.212.212:38213
02:06:40.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:40.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:40.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:40.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:40.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.212.254:46217,127.0.212.253:32829,127.0.212.252:41741
02:06:40.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:40.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38193/jwks.json
02:06:40.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:06:40.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:06:40.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:40.479497 1853 sys_catalog.cc:263] Verifying existing consensus state
02:06:40.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:40.490269 1853 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f: Bootstrap starting.
02:06:40.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:40.554629 1853 log.cc:826] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f: Log is configured to *not* fsync() on all Append() calls
02:06:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:40.601234 1858 negotiation.cc:337] Failed RPC negotiation. Trace:
02:06:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 02:06:40.557483 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.212.194:51537 (local address 127.0.212.254:46217)
02:06:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 02:06:40.579253 (+ 21770us) server_negotiation.cc:206] Beginning negotiation
02:06:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 02:06:40.579284 (+ 31us) server_negotiation.cc:399] Waiting for connection header
02:06:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 02:06:40.590199 (+ 10915us) server_negotiation.cc:407] Connection header received
02:06:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 02:06:40.596572 (+ 6373us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
02:06:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 02:06:40.596621 (+ 49us) server_negotiation.cc:461] Received NEGOTIATE request from client
02:06:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 02:06:40.597125 (+ 504us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
02:06:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 02:06:40.597544 (+ 419us) server_negotiation.cc:226] Negotiated authn=SASL
02:06:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 02:06:40.600040 (+ 2496us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.212.194:51537: BlockingRecv error: recv got EOF from 127.0.212.194:51537 (error 108)
02:06:40.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":20631,"thread_start_us":20716,"threads_started":1}
02:06:40.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:40.607872 1853 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:06:40.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:40.609014 1853 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f: Bootstrap complete.
02:06:40.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:40.638859 1853 raft_consensus.cc:357] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } }
02:06:40.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:40.642585 1853 raft_consensus.cc:738] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: d068f217170340f9b4955443ddfda35f, State: Initialized, Role: FOLLOWER
02:06:40.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:40.643716 1853 consensus_queue.cc:260] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } }
02:06:40.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:40.647440 1861 sys_catalog.cc:455] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } } }
02:06:40.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:40.648625 1861 sys_catalog.cc:458] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [sys.catalog]: This master's current role is: FOLLOWER
02:06:40.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:40.649605 1853 sys_catalog.cc:564] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [sys.catalog]: configured and running, proceeding with master startup.
02:06:40.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:40.725725 1872 catalog_manager.cc:1261] Loaded cluster ID: 0d51723196934732bbe6c094bcb5b19d
02:06:40.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:40.727156 1872 catalog_manager.cc:1554] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f: loading cluster ID for follower catalog manager: success
02:06:40.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:40 dist-test-slave-szd5 krb5kdc[863](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778800, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.254@KRBTEST.COM for kudu/127.0.212.254@KRBTEST.COM
02:06:40.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:40.744103 1872 catalog_manager.cc:1576] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f: acquiring CA information for follower catalog manager: success
02:06:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:40.750252 1872 catalog_manager.cc:1604] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:06:40.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:40.770799 1743 master_runner.cc:186] Error getting master registration for 127.0.212.253:32829: Network error: Client connection negotiation failed: client connection to 127.0.212.253:32829: connect: Connection refused (error 111),
02:06:40.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:40.797516 1743 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:06:40.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:40 dist-test-slave-szd5 krb5kdc[863](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778800, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.194@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:40.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:40.838346 1376 init.cc:295] Successfully reacquired a new kerberos TGT
02:06:41.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:41.023094 1855 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:06:41.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:41.023937 1855 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:06:41.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:41.025068 1855 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:06:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:41.084223 1855 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:06:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:41.084590 1855 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:06:41.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:41.085130 1855 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:06:41.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:41.085477 1855 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:06:41.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:41.085809 1855 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:06:41.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:41.086120 1855 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:06:41.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:41.086361 1855 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38193/jwks.json
02:06:41.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:41.124552 1342 heartbeater.cc:344] Connected to a master server at 127.0.212.254:46217
02:06:41.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:41.133558 1818 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "ce46aaca229a4a5a9e62b9338cac9b42" instance_seqno: 1756778786352500) as {username='kudu', principal='kudu/127.0.212.193@KRBTEST.COM'} at 127.0.212.193:56919; Asking this server to re-register.
02:06:41.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:41.135284 1342 heartbeater.cc:461] Registering TS with master...
02:06:41.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:41.139494 1818 ts_manager.cc:194] Registered new tserver with Master: ce46aaca229a4a5a9e62b9338cac9b42 (127.0.212.193:44097)
02:06:41.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:41.174961 1855 master_runner.cc:386] Master server non-default flags:
02:06:41.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:41.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.212.212:38213
02:06:41.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:41.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:41.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/data
02:06:41.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/wal
02:06:41.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:06:41.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:41.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.212.254:46217,127.0.212.253:32829,127.0.212.252:41741
02:06:41.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab
02:06:41.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.212.253
02:06:41.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:41.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:41.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:41.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.212.253:32829
02:06:41.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:41.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:06:41.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38193/jwks.json
02:06:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:06:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:06:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/data/info.pb
02:06:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.212.253
02:06:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39491
02:06:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:06:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:06:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:06:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/logs
02:06:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:06:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:06:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:06:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:06:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
02:06:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:06:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 01:43:19 UTC on 5fd53c4cbb9d
02:06:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7857
02:06:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:06:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:41.176702 1855 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:06:41.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:41.178048 1855 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:06:41.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:41.208664 1883 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:41.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:41.249447 1647 heartbeater.cc:344] Connected to a master server at 127.0.212.254:46217
02:06:41.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:41.253104 1818 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "8f26756046ee4fb0a2e55f50a20e45ab" instance_seqno: 1756778794891758) as {username='kudu', principal='kudu/127.0.212.195@KRBTEST.COM'} at 127.0.212.195:50069; Asking this server to re-register.
02:06:41.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:41.254812 1647 heartbeater.cc:461] Registering TS with master...
02:06:41.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:41.258605 1818 ts_manager.cc:194] Registered new tserver with Master: 8f26756046ee4fb0a2e55f50a20e45ab (127.0.212.195:38071)
02:06:41.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:41.363090 1888 raft_consensus.cc:491] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:06:41.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:41.367897 1888 raft_consensus.cc:513] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } }
02:06:41.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:41.386997 1888 leader_election.cc:290] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers d068f217170340f9b4955443ddfda35f (127.0.212.254:46217), ced15ce4e97b4e92baa9efefcecd6a36 (127.0.212.253:32829)
02:06:41.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:41.400233 1828 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "bdab70b6529844f5a3f4d6f008ba1079" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "d068f217170340f9b4955443ddfda35f" is_pre_election: true
02:06:41.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:41.401758 1828 raft_consensus.cc:2466] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate bdab70b6529844f5a3f4d6f008ba1079 in term 1.
02:06:41.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:41.404814 1691 leader_election.cc:304] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: bdab70b6529844f5a3f4d6f008ba1079, d068f217170340f9b4955443ddfda35f; no voters:
02:06:41.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:41.405938 1888 raft_consensus.cc:2802] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 1 FOLLOWER]: Leader pre-election won for term 2
02:06:41.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:41.406287 1888 raft_consensus.cc:491] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:06:41.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:41.406631 1888 raft_consensus.cc:3058] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 1 FOLLOWER]: Advancing to term 2
02:06:41.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:41.427359 1693 leader_election.cc:336] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer ced15ce4e97b4e92baa9efefcecd6a36 (127.0.212.253:32829): Network error: Client connection negotiation failed: client connection to 127.0.212.253:32829: connect: Connection refused (error 111)
02:06:41.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:41.430249 1888 raft_consensus.cc:513] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } }
02:06:41.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:41.434041 1828 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "bdab70b6529844f5a3f4d6f008ba1079" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "d068f217170340f9b4955443ddfda35f"
02:06:41.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:41.434211 1888 leader_election.cc:290] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [CANDIDATE]: Term 2 election: Requested vote from peers d068f217170340f9b4955443ddfda35f (127.0.212.254:46217), ced15ce4e97b4e92baa9efefcecd6a36 (127.0.212.253:32829)
02:06:41.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:41.434705 1828 raft_consensus.cc:3058] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [term 1 FOLLOWER]: Advancing to term 2
02:06:41.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:41.438997 1693 leader_election.cc:336] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer ced15ce4e97b4e92baa9efefcecd6a36 (127.0.212.253:32829): Network error: Client connection negotiation failed: client connection to 127.0.212.253:32829: connect: Connection refused (error 111)
02:06:41.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:41.446092 1828 raft_consensus.cc:2466] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate bdab70b6529844f5a3f4d6f008ba1079 in term 2.
02:06:41.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:41.447155 1691 leader_election.cc:304] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: bdab70b6529844f5a3f4d6f008ba1079, d068f217170340f9b4955443ddfda35f; no voters: ced15ce4e97b4e92baa9efefcecd6a36
02:06:41.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:41.448563 1888 raft_consensus.cc:2802] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 2 FOLLOWER]: Leader election won for term 2
02:06:41.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:41.465646 1888 raft_consensus.cc:695] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 2 LEADER]: Becoming Leader. State: Replica: bdab70b6529844f5a3f4d6f008ba1079, State: Running, Role: LEADER
02:06:41.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:41.467024 1888 consensus_queue.cc:237] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } }
02:06:41.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:41.490782 1892 sys_catalog.cc:455] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [sys.catalog]: SysCatalogTable state changed. Reason: New leader bdab70b6529844f5a3f4d6f008ba1079. Latest consensus state: current_term: 2 leader_uuid: "bdab70b6529844f5a3f4d6f008ba1079" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } } }
02:06:41.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:41.492628 1892 sys_catalog.cc:458] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [sys.catalog]: This master's current role is: LEADER
02:06:41.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:41.506006 1894 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
02:06:41.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:41.533210 1894 catalog_manager.cc:1486] Initializing Kudu cluster ID...
02:06:41.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:41.555776 1894 catalog_manager.cc:1261] Loaded cluster ID: 0d51723196934732bbe6c094bcb5b19d
02:06:41.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:41.556579 1894 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
02:06:41.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:41.564985 1894 catalog_manager.cc:1506] Loading token signing keys...
02:06:41.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:41.572289 1894 catalog_manager.cc:5966] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079: Loaded TSK: 0
02:06:41.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:41.579420 1894 catalog_manager.cc:1516] Initializing in-progress tserver states...
02:06:41.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:41.640810 1500 heartbeater.cc:344] Connected to a master server at 127.0.212.254:46217
02:06:41.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:41.644546 1818 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "4ac25f02a47d4bc2a47eb802eb602ee9" instance_seqno: 1756778791257506) as {username='kudu', principal='kudu/127.0.212.194@KRBTEST.COM'} at 127.0.212.194:35051; Asking this server to re-register.
02:06:41.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:41.646377 1500 heartbeater.cc:461] Registering TS with master...
02:06:41.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:41.651805 1818 ts_manager.cc:194] Registered new tserver with Master: 4ac25f02a47d4bc2a47eb802eb602ee9 (127.0.212.194:39417)
02:06:42.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:42.017935 1828 raft_consensus.cc:1273] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [term 2 FOLLOWER]: Refusing update from remote peer bdab70b6529844f5a3f4d6f008ba1079: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
02:06:42.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:42.020227 1898 consensus_queue.cc:1035] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.001s
02:06:42.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:42.052284 1899 sys_catalog.cc:455] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [sys.catalog]: SysCatalogTable state changed. Reason: New leader bdab70b6529844f5a3f4d6f008ba1079. Latest consensus state: current_term: 2 leader_uuid: "bdab70b6529844f5a3f4d6f008ba1079" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } } }
02:06:42.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:42.064949 1899 sys_catalog.cc:458] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [sys.catalog]: This master's current role is: FOLLOWER
02:06:42.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:42.084548 1899 sys_catalog.cc:455] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "bdab70b6529844f5a3f4d6f008ba1079" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } } }
02:06:42.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:42.086064 1899 sys_catalog.cc:458] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [sys.catalog]: This master's current role is: FOLLOWER
02:06:42.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:42.092548 1901 sys_catalog.cc:455] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "bdab70b6529844f5a3f4d6f008ba1079" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } } }
02:06:42.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:42.093780 1901 sys_catalog.cc:458] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [sys.catalog]: This master's current role is: LEADER
02:06:42.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:42.119246 1693 consensus_peers.cc:489] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 -> Peer ced15ce4e97b4e92baa9efefcecd6a36 (127.0.212.253:32829): Couldn't send request to peer ced15ce4e97b4e92baa9efefcecd6a36. Status: Network error: Client connection negotiation failed: client connection to 127.0.212.253:32829: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
02:06:43.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.141083 1338 heartbeater.cc:499] Master 127.0.212.252:41741 was elected leader, sending a full tablet report...
02:06:43.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:41.214419 1885 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:43.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:42.601951 1881 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 1855
02:06:43.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:43.177073 1855 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.965s user 0.539s sys 1.184s
02:06:43.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:43.179140 1855 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.967s user 0.539s sys 1.184s
02:06:43.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:43.191171 1917 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:43.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.191373 1855 server_base.cc:1047] running on GCE node
02:06:43.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.192917 1855 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:06:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.196269 1855 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:06:43.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.197690 1855 hybrid_clock.cc:648] HybridClock initialized: now 1756778803197588 us; error 90 us; skew 500 ppm
02:06:43.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.198866 1855 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:06:43.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:43 dist-test-slave-szd5 krb5kdc[863](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778803, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:43.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.207986 1855 init.cc:377] Logged in from keytab as kudu/127.0.212.253@KRBTEST.COM (short username kudu)
02:06:43.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.214818 1855 webserver.cc:480] Webserver started at http://127.0.212.253:39491/ using document root <none> and password file <none>
02:06:43.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.216041 1855 fs_manager.cc:362] Metadata directory not provided
02:06:43.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.216336 1855 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:06:43.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.223153 1643 heartbeater.cc:499] Master 127.0.212.252:41741 was elected leader, sending a full tablet report...
02:06:43.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.228284 1855 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.007s sys 0.000s
02:06:43.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.234306 1923 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:06:43.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.235389 1855 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.003s
02:06:43.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.235742 1855 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/data,/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/wal
02:06:43.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ced15ce4e97b4e92baa9efefcecd6a36"
02:06:43.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 02:06:17 on dist-test-slave-szd5"
02:06:43.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.237697 1855 fs_report.cc:389] FS layout report
02:06:43.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:06:43.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/wal
02:06:43.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/wal
02:06:43.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/data/data
02:06:43.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:06:43.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:06:43.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:06:43.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:06:43.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:06:43.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:06:43.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:06:43.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:06:43.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:06:43.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:06:43.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:43.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.250836 1658 master_runner.cc:186] Error getting master registration for 127.0.212.253:32829: Network error: Client connection negotiation failed: client connection to 127.0.212.253:32829: connect: Connection refused (error 111),
02:06:43.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.251341 1658 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:06:43.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.322415 1855 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:06:43.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.338383 1855 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:43.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.338944 1855 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:43.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.510563 1855 rpc_server.cc:307] RPC server started. Bound to: 127.0.212.253:32829
02:06:43.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.511070 1975 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.212.253:32829 every 8 connection(s)
02:06:43.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.514400 1855 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/data/info.pb
02:06:43.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.521481 851 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu as pid 1855
02:06:43.522 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:06:43.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.538441 1976 sys_catalog.cc:263] Verifying existing consensus state
02:06:43.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.546417 1976 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36: Bootstrap starting.
02:06:43.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.593505 1494 heartbeater.cc:499] Master 127.0.212.252:41741 was elected leader, sending a full tablet report...
02:06:43.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.615082 1976 log.cc:826] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36: Log is configured to *not* fsync() on all Append() calls
02:06:43.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:43 dist-test-slave-szd5 krb5kdc[863](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778797, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.252@KRBTEST.COM for kudu/127.0.212.253@KRBTEST.COM
02:06:43.622 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.0.212.252:41741(127.0.212.252:41741)] connecting to peer
02:06:43.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.640185 1976 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:06:43.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.641312 1976 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36: Bootstrap complete.
02:06:43.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.671150 1976 raft_consensus.cc:357] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } }
02:06:43.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.675151 1976 raft_consensus.cc:738] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: ced15ce4e97b4e92baa9efefcecd6a36, State: Initialized, Role: FOLLOWER
02:06:43.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.676311 1976 consensus_queue.cc:260] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } }
02:06:43.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.694398 1982 sys_catalog.cc:455] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } } }
02:06:43.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.695257 1982 sys_catalog.cc:458] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36 [sys.catalog]: This master's current role is: FOLLOWER
02:06:43.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.696520 1976 sys_catalog.cc:564] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36 [sys.catalog]: configured and running, proceeding with master startup.
02:06:43.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.697257 1951 raft_consensus.cc:3058] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36 [term 1 FOLLOWER]: Advancing to term 2
02:06:43.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:43 dist-test-slave-szd5 krb5kdc[863](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778800, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.194@KRBTEST.COM for kudu/127.0.212.253@KRBTEST.COM
02:06:43.756 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.0.212.254:46217(127.0.212.254:46217)] connecting to peer
02:06:43.758 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.0.212.253:32829(127.0.212.253:32829)] connecting to peer
02:06:43.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.767030 1495 heartbeater.cc:344] Connected to a master server at 127.0.212.253:32829
02:06:43.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.790524 1992 sys_catalog.cc:455] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "bdab70b6529844f5a3f4d6f008ba1079" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } } }
02:06:43.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.793380 1992 sys_catalog.cc:458] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36 [sys.catalog]: This master's current role is: FOLLOWER
02:06:43.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.797461 1982 sys_catalog.cc:455] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36 [sys.catalog]: SysCatalogTable state changed. Reason: New leader bdab70b6529844f5a3f4d6f008ba1079. Latest consensus state: current_term: 2 leader_uuid: "bdab70b6529844f5a3f4d6f008ba1079" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } } }
02:06:43.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.798306 1982 sys_catalog.cc:458] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36 [sys.catalog]: This master's current role is: FOLLOWER
02:06:43.816 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.0.212.254:46217(127.0.212.254:46217)] Successfully connected to peer
02:06:43.818 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.0.212.252:41741(127.0.212.252:41741)] Successfully connected to peer
02:06:43.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.849639 2000 catalog_manager.cc:1261] Loaded cluster ID: 0d51723196934732bbe6c094bcb5b19d
02:06:43.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.849958 2000 catalog_manager.cc:1554] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36: loading cluster ID for follower catalog manager: success
02:06:43.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:43 dist-test-slave-szd5 krb5kdc[863](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778803, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.253@KRBTEST.COM for kudu/127.0.212.254@KRBTEST.COM
02:06:43.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.862439 2000 catalog_manager.cc:1576] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36: acquiring CA information for follower catalog manager: success
02:06:43.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:43.870541 2000 catalog_manager.cc:1604] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:06:43.948 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.0.212.253:32829(127.0.212.253:32829)] Successfully connected to peer
02:06:43.971 [WARN - kudu-nio-0] (SecurityContext.java:206) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
02:06:43.990 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.0.212.254
02:06:43.991 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.0.212.252
02:06:44.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:44.094962 1901 sys_catalog.cc:455] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "bdab70b6529844f5a3f4d6f008ba1079" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } } }
02:06:44.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:44.096233 1901 sys_catalog.cc:458] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [sys.catalog]: This master's current role is: LEADER
02:06:44.132 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.0.212.253
02:06:44.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:44.211349 1339 heartbeater.cc:344] Connected to a master server at 127.0.212.253:32829
02:06:44.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:44.221237 1941 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "ce46aaca229a4a5a9e62b9338cac9b42" instance_seqno: 1756778786352500) as {username='kudu', principal='kudu/127.0.212.193@KRBTEST.COM'} at 127.0.212.193:46243; Asking this server to re-register.
02:06:44.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:44.223371 1339 heartbeater.cc:461] Registering TS with master...
02:06:44.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:44.234691 1941 ts_manager.cc:194] Registered new tserver with Master: ce46aaca229a4a5a9e62b9338cac9b42 (127.0.212.193:44097)
02:06:44.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:44 dist-test-slave-szd5 krb5kdc[863](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778795, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.0.212.253@KRBTEST.COM
02:06:44.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:44 dist-test-slave-szd5 krb5kdc[863](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778795, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.0.212.254@KRBTEST.COM
02:06:44.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:44.312872 1644 heartbeater.cc:344] Connected to a master server at 127.0.212.253:32829
02:06:44.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:44.316334 1941 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "8f26756046ee4fb0a2e55f50a20e45ab" instance_seqno: 1756778794891758) as {username='kudu', principal='kudu/127.0.212.195@KRBTEST.COM'} at 127.0.212.195:49501; Asking this server to re-register.
02:06:44.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:44.317945 1644 heartbeater.cc:461] Registering TS with master...
02:06:44.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:44.321750 1941 ts_manager.cc:194] Registered new tserver with Master: 8f26756046ee4fb0a2e55f50a20e45ab (127.0.212.195:38071)
02:06:44.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:44 dist-test-slave-szd5 krb5kdc[863](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778795, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.0.212.252@KRBTEST.COM
02:06:44.355 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x2814abea, L:/127.0.0.1:43852 - R:/127.0.212.254:46217] using SASL/GSSAPI
02:06:44.375 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xc2fa1689, L:/127.0.0.1:53128 - R:/127.0.212.253:32829] using SASL/GSSAPI
02:06:44.383 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xf478df26, L:/127.0.0.1:35754 - R:/127.0.212.252:41741] using SASL/GSSAPI
02:06:44.425 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
02:06:44.428 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=901), Traces: [0ms] refreshing cache from master, [66ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.212.252:41741, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.212.254:46217, [224ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.212.253:32829, [845ms] Sub RPC ConnectToMaster: received response from server master-127.0.212.254:46217: OK, [851ms] Sub RPC ConnectToMaster: received response from server master-127.0.212.253:32829: OK, [860ms] Sub RPC ConnectToMaster: received response from server master-127.0.212.252:41741: OK, deferred=null) after lookup
02:06:44.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:44 dist-test-slave-szd5 krb5kdc[863](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778804, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.195@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:44.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:44.489869 1526 init.cc:295] Successfully reacquired a new kerberos TGT
02:06:44.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:44.775981 1941 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "4ac25f02a47d4bc2a47eb802eb602ee9" instance_seqno: 1756778791257506) as {username='kudu', principal='kudu/127.0.212.194@KRBTEST.COM'} at 127.0.212.194:36967; Asking this server to re-register.
02:06:44.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:44.777156 1495 heartbeater.cc:461] Registering TS with master...
02:06:44.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:44.779656 1941 ts_manager.cc:194] Registered new tserver with Master: 4ac25f02a47d4bc2a47eb802eb602ee9 (127.0.212.194:39417)
02:06:45.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:45 dist-test-slave-szd5 krb5kdc[863](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778805, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.193@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:45.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:45.273255 1217 init.cc:295] Successfully reacquired a new kerberos TGT
02:06:45.458 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
02:06:45.459 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
02:06:45.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:45 dist-test-slave-szd5 krb5kdc[863](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778805, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:45.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
02:06:45.500 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:06:45.500 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/krb5cc
02:06:45.502 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:06:45.502 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.0.212.252:41741
02:06:45.502 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
02:06:45.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:45.503163 851 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu with pid 1658
02:06:45.550 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:06:45.551 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.0.212.252:41741(127.0.212.252:41741)] handling channelInactive
02:06:45.551 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.0.212.254:46217
02:06:45.551 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
02:06:45.551 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.212.252:41741(127.0.212.252:41741)] cleaning up while in state READY due to: connection closed
02:06:45.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:45.553761 851 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu with pid 1743
02:06:45.585 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:06:45.586 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.0.212.253:32829
02:06:45.586 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.0.212.254:46217(127.0.212.254:46217)] handling channelInactive
02:06:45.586 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.0.212.254:46217(127.0.212.254:46217)] cleaning up while in state READY due to: connection closed
02:06:45.586 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
02:06:45.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:45.586568 851 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu with pid 1855
02:06:45.615 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:06:45.615 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.0.212.252:41741
02:06:45.615 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
02:06:45.616 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.0.212.253:32829(127.0.212.253:32829)] handling channelInactive
02:06:45.616 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.212.253:32829(127.0.212.253:32829)] cleaning up while in state READY due to: connection closed
02:06:45.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:45.616899 851 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu
02:06:45.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu
02:06:45.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/wal
02:06:45.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/data
02:06:45.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:06:45.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:06:45.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:06:45.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:45.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:45.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:45.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/logs
02:06:45.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/data/info.pb
02:06:45.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:45.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:06:45.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:06:45.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:06:45.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:06:45.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:45.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:45.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:45.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:06:45.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:06:45.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:45.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:45.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.212.252:41741
02:06:45.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.212.252
02:06:45.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33899
02:06:45.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab
02:06:45.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.212.252
02:06:45.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:45.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:06:45.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:06:45.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:06:45.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.212.212:38213
02:06:45.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:45.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:45.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:45.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:45.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.212.254:46217,127.0.212.253:32829,127.0.212.252:41741
02:06:45.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:45.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38193/jwks.json
02:06:45.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:06:45.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:06:45.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:45.933410 2018 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:06:45.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:45.934190 2018 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:06:45.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:45.934957 2018 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:06:45.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:45.968377 2018 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:06:45.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:45.968626 2018 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:06:45.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:45.969038 2018 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:06:45.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:45.969265 2018 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:06:45.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:45.969461 2018 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:06:45.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:45.969648 2018 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:06:45.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:45.969781 2018 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38193/jwks.json
02:06:46.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:46.010604 2018 master_runner.cc:386] Master server non-default flags:
02:06:46.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:46.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.212.212:38213
02:06:46.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:46.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:46.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/data
02:06:46.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/wal
02:06:46.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:06:46.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:46.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.212.254:46217,127.0.212.253:32829,127.0.212.252:41741
02:06:46.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab
02:06:46.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.212.252
02:06:46.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:46.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:46.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:46.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.212.252:41741
02:06:46.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:46.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:06:46.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:46.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38193/jwks.json
02:06:46.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:46.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:06:46.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:06:46.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:46.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:46.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/data/info.pb
02:06:46.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.212.252
02:06:46.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33899
02:06:46.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:06:46.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:46.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:06:46.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:06:46.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:46.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/logs
02:06:46.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:46.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:06:46.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:06:46.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:06:46.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:06:46.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
02:06:46.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:06:46.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 01:43:19 UTC on 5fd53c4cbb9d
02:06:46.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7857
02:06:46.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:06:46.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:46.011897 2018 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:06:46.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:46.012877 2018 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:06:46.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:46.023568 2024 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:46.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:46.149416 352 debug-util.cc:398] Leaking SignalData structure 0x7b08000a9120 after lost signal to thread 32751
02:06:46.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:46.150408 352 debug-util.cc:398] Leaking SignalData structure 0x7b080006f360 after lost signal to thread 355
02:06:48.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:47.427194 2023 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 2018
02:06:48.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:48.213073 2023 kernel_stack_watchdog.cc:198] Thread 2018 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 401ms:
02:06:48.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:06:48.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:06:48.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:06:48.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:06:48.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:46.023859 2025 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:48.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:48.216861 2027 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:48.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:48.217748 2026 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2188 milliseconds
02:06:48.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:48.216441 2018 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.193s user 0.578s sys 1.216s
02:06:48.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:48.223384 2018 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.200s user 0.579s sys 1.217s
02:06:48.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:48.223938 2018 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:06:48.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:48.225150 2018 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:06:48.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:48.231019 2018 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:06:48.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:48.232460 2018 hybrid_clock.cc:648] HybridClock initialized: now 1756778808232328 us; error 135 us; skew 500 ppm
02:06:48.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:48.233323 2018 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:06:48.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:48 dist-test-slave-szd5 krb5kdc[863](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778808, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:48.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:48.239946 2018 init.cc:377] Logged in from keytab as kudu/127.0.212.252@KRBTEST.COM (short username kudu)
02:06:48.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:48.246919 2018 webserver.cc:480] Webserver started at http://127.0.212.252:33899/ using document root <none> and password file <none>
02:06:48.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:48.248234 2018 fs_manager.cc:362] Metadata directory not provided
02:06:48.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:48.248795 2018 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:06:48.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:48.258467 2018 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.001s sys 0.005s
02:06:48.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:48.263664 2048 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:06:48.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:48.264761 2018 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
02:06:48.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:48.265056 2018 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/data,/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/wal
02:06:48.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bdab70b6529844f5a3f4d6f008ba1079"
02:06:48.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 02:06:20 on dist-test-slave-szd5"
02:06:48.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:48.266952 2018 fs_report.cc:389] FS layout report
02:06:48.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:06:48.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/wal
02:06:48.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/wal
02:06:48.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/data/data
02:06:48.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:06:48.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:06:48.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:06:48.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:06:48.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:06:48.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:06:48.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:06:48.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:06:48.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:06:48.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:06:48.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:48.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:48.326871 2018 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:06:48.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:48.336091 2018 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:48.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:48.336629 2018 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:48.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:48.429448 2018 rpc_server.cc:307] RPC server started. Bound to: 127.0.212.252:41741
02:06:48.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:48.429533 2102 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.212.252:41741 every 8 connection(s)
02:06:48.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:48.434253 2018 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/data/info.pb
02:06:48.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:48.439429 851 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu as pid 2018
02:06:48.440 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:06:48.440 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.0.212.254:46217
02:06:48.440 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
02:06:48.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:48.442646 851 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu
02:06:48.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu
02:06:48.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/wal
02:06:48.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/data
02:06:48.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:06:48.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:06:48.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:06:48.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:48.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:48.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:48.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/logs
02:06:48.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/data/info.pb
02:06:48.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:48.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:06:48.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:06:48.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:06:48.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:06:48.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:48.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:48.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:48.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:06:48.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:06:48.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:48.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:48.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.212.254:46217
02:06:48.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.212.254
02:06:48.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44809
02:06:48.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab
02:06:48.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.212.254
02:06:48.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:48.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:06:48.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:06:48.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:06:48.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.212.212:38213
02:06:48.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:48.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:48.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:48.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:48.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.212.254:46217,127.0.212.253:32829,127.0.212.252:41741
02:06:48.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:48.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38193/jwks.json
02:06:48.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:06:48.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:06:48.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:48.474684 2103 sys_catalog.cc:263] Verifying existing consensus state
02:06:48.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:48.482590 2103 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079: Bootstrap starting.
02:06:48.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:48.529902 2103 log.cc:826] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079: Log is configured to *not* fsync() on all Append() calls
02:06:48.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:48.547061 2103 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:06:48.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:48.547945 2103 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079: Bootstrap complete.
02:06:48.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:48.570722 2103 raft_consensus.cc:357] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } }
02:06:48.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:48.573172 2103 raft_consensus.cc:738] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: bdab70b6529844f5a3f4d6f008ba1079, State: Initialized, Role: FOLLOWER
02:06:48.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:48.574184 2103 consensus_queue.cc:260] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } }
02:06:48.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:48.577286 2107 sys_catalog.cc:455] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } } }
02:06:48.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:48.579073 2107 sys_catalog.cc:458] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [sys.catalog]: This master's current role is: FOLLOWER
02:06:48.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:48.579241 2103 sys_catalog.cc:564] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [sys.catalog]: configured and running, proceeding with master startup.
02:06:48.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:48.641217 2118 catalog_manager.cc:1261] Loaded cluster ID: 0d51723196934732bbe6c094bcb5b19d
02:06:48.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:48.642004 2118 catalog_manager.cc:1554] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079: loading cluster ID for follower catalog manager: success
02:06:48.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:48.653637 2118 catalog_manager.cc:1576] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079: acquiring CA information for follower catalog manager: success
02:06:48.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:48.660225 2118 catalog_manager.cc:1604] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:06:48.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:48.616743 2018 master_runner.cc:186] Error getting master registration for 127.0.212.254:46217: Network error: Client connection negotiation failed: client connection to 127.0.212.254:46217: connect: Connection refused (error 111),
02:06:48.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:48.661346 2018 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:06:48.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:48.716010 1494 heartbeater.cc:344] Connected to a master server at 127.0.212.252:41741
02:06:48.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:48.723181 2068 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "4ac25f02a47d4bc2a47eb802eb602ee9" instance_seqno: 1756778791257506) as {username='kudu', principal='kudu/127.0.212.194@KRBTEST.COM'} at 127.0.212.194:45653; Asking this server to re-register.
02:06:48.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:48.729871 1494 heartbeater.cc:461] Registering TS with master...
02:06:48.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:48.734014 2067 ts_manager.cc:194] Registered new tserver with Master: 4ac25f02a47d4bc2a47eb802eb602ee9 (127.0.212.194:39417)
02:06:48.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:48.986656 2105 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:06:48.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:48.987571 2105 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:06:48.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:48.994094 2105 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:06:49.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:49.063052 2105 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:06:49.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:49.063436 2105 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:06:49.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:49.063967 2105 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:06:49.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:49.064325 2105 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:06:49.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:49.064700 2105 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:06:49.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:49.065048 2105 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:06:49.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:49.065302 2105 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38193/jwks.json
02:06:49.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:49.138293 2105 master_runner.cc:386] Master server non-default flags:
02:06:49.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:49.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.212.212:38213
02:06:49.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:49.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:49.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/data
02:06:49.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/wal
02:06:49.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:06:49.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:49.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.212.254:46217,127.0.212.253:32829,127.0.212.252:41741
02:06:49.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab
02:06:49.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.212.254
02:06:49.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:49.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:49.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:49.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.212.254:46217
02:06:49.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:49.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:06:49.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:49.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38193/jwks.json
02:06:49.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:49.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:06:49.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:06:49.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:49.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:49.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/data/info.pb
02:06:49.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.212.254
02:06:49.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44809
02:06:49.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:06:49.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:49.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:06:49.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:06:49.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:49.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/logs
02:06:49.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:49.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:06:49.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:06:49.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:06:49.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:06:49.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
02:06:49.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:06:49.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 01:43:19 UTC on 5fd53c4cbb9d
02:06:49.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7857
02:06:49.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:06:49.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:49.140123 2105 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:06:49.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:49.141393 2105 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:06:49.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:49.166770 2130 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:49.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:49.248642 1338 heartbeater.cc:344] Connected to a master server at 127.0.212.252:41741
02:06:49.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:49.251753 2068 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "ce46aaca229a4a5a9e62b9338cac9b42" instance_seqno: 1756778786352500) as {username='kudu', principal='kudu/127.0.212.193@KRBTEST.COM'} at 127.0.212.193:48769; Asking this server to re-register.
02:06:49.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:49.253480 1338 heartbeater.cc:461] Registering TS with master...
02:06:49.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:49.257578 2068 ts_manager.cc:194] Registered new tserver with Master: ce46aaca229a4a5a9e62b9338cac9b42 (127.0.212.193:44097)
02:06:49.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:49.319010 1643 heartbeater.cc:344] Connected to a master server at 127.0.212.252:41741
02:06:49.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:49.323426 2068 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "8f26756046ee4fb0a2e55f50a20e45ab" instance_seqno: 1756778794891758) as {username='kudu', principal='kudu/127.0.212.195@KRBTEST.COM'} at 127.0.212.195:33493; Asking this server to re-register.
02:06:49.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:49.325934 1643 heartbeater.cc:461] Registering TS with master...
02:06:49.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:49.336160 2068 ts_manager.cc:194] Registered new tserver with Master: 8f26756046ee4fb0a2e55f50a20e45ab (127.0.212.195:38071)
02:06:49.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:49.973387 2139 raft_consensus.cc:491] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:06:49.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:49.974185 2139 raft_consensus.cc:513] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } }
02:06:50.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:50.008266 2139 leader_election.cc:290] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers d068f217170340f9b4955443ddfda35f (127.0.212.254:46217), ced15ce4e97b4e92baa9efefcecd6a36 (127.0.212.253:32829)
02:06:50.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:50.058842 2055 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.212.253:32829: connect: Connection refused (error 111)
02:06:50.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:50.067974 2053 leader_election.cc:336] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer d068f217170340f9b4955443ddfda35f (127.0.212.254:46217): Network error: Client connection negotiation failed: client connection to 127.0.212.254:46217: connect: Connection refused (error 111)
02:06:50.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:50.071841 2055 leader_election.cc:336] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer ced15ce4e97b4e92baa9efefcecd6a36 (127.0.212.253:32829): Network error: Client connection negotiation failed: client connection to 127.0.212.253:32829: connect: Connection refused (error 111)
02:06:50.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:50.072264 2055 leader_election.cc:304] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: bdab70b6529844f5a3f4d6f008ba1079; no voters: ced15ce4e97b4e92baa9efefcecd6a36, d068f217170340f9b4955443ddfda35f
02:06:50.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:50.073065 2139 raft_consensus.cc:2747] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
02:06:50.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:50.566753 2129 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 2105
02:06:50.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:49.168363 2131 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:50.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:50.711335 2129 kernel_stack_watchdog.cc:198] Thread 2105 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 396ms:
02:06:50.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:06:50.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:06:50.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:06:50.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:06:50.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:50.711905 2105 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.546s user 0.585s sys 0.918s
02:06:50.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:50.714190 2105 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.546s user 0.585s sys 0.918s
02:06:50.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:50.726500 2105 server_base.cc:1047] running on GCE node
02:06:50.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:50.728081 2148 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:50.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:50.729305 2105 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:06:50.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:50.743355 2105 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:06:50.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:50.746307 2105 hybrid_clock.cc:648] HybridClock initialized: now 1756778810746159 us; error 170 us; skew 500 ppm
02:06:50.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:50.747756 2105 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:06:50.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:50 dist-test-slave-szd5 krb5kdc[863](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778810, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:50.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:50.755308 2105 init.cc:377] Logged in from keytab as kudu/127.0.212.254@KRBTEST.COM (short username kudu)
02:06:50.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:50.761567 2105 webserver.cc:480] Webserver started at http://127.0.212.254:44809/ using document root <none> and password file <none>
02:06:50.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:50.762521 2105 fs_manager.cc:362] Metadata directory not provided
02:06:50.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:50.762718 2105 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:06:50.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:50.770977 2105 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.006s sys 0.000s
02:06:50.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:50.776430 2155 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:06:50.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:50.777536 2105 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
02:06:50.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:50.777873 2105 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/data,/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/wal
02:06:50.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d068f217170340f9b4955443ddfda35f"
02:06:50.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 02:06:14 on dist-test-slave-szd5"
02:06:50.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:50.780350 2105 fs_report.cc:389] FS layout report
02:06:50.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:06:50.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/wal
02:06:50.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/wal
02:06:50.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/data/data
02:06:50.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:06:50.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:06:50.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:06:50.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:06:50.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:06:50.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:06:50.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:06:50.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:06:50.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:06:50.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:06:50.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:50.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:50 dist-test-slave-szd5 krb5kdc[863](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778810, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.194@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:50.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:50.842308 1376 init.cc:295] Successfully reacquired a new kerberos TGT
02:06:50.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:50.844870 2105 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:06:50.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:50.855197 2105 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:50.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:50.855726 2105 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:50.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:50.947609 2105 rpc_server.cc:307] RPC server started. Bound to: 127.0.212.254:46217
02:06:50.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:50.947764 2206 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.212.254:46217 every 8 connection(s)
02:06:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:50.950609 2105 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/data/info.pb
02:06:50.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:50.958436 851 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu as pid 2105
02:06:50.959 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:06:50.960 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.0.212.253:32829
02:06:50.960 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
02:06:50.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:50.962082 2207 sys_catalog.cc:263] Verifying existing consensus state
02:06:50.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:50.962603 851 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu
02:06:50.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu
02:06:50.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/wal
02:06:50.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/data
02:06:50.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:06:50.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:06:50.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:06:50.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:50.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:50.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:50.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/logs
02:06:50.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/data/info.pb
02:06:50.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:50.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:06:50.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:06:50.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:06:50.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:06:50.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:50.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:50.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:50.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:06:50.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:06:50.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:50.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:50.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.212.253:32829
02:06:50.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.212.253
02:06:50.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39491
02:06:50.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab
02:06:50.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.212.253
02:06:50.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:50.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:06:50.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:06:50.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:06:50.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.212.212:38213
02:06:50.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:50.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:50.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:50.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:50.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.212.254:46217,127.0.212.253:32829,127.0.212.252:41741
02:06:50.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:50.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38193/jwks.json
02:06:50.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:06:50.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:06:50.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:50.967984 2207 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f: Bootstrap starting.
02:06:51.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:51.024062 2207 log.cc:826] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f: Log is configured to *not* fsync() on all Append() calls
02:06:51.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:51.047906 2207 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:06:51.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:51.049072 2207 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f: Bootstrap complete.
02:06:51.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:51.083003 2207 raft_consensus.cc:357] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } }
02:06:51.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:51.086973 2207 raft_consensus.cc:738] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: d068f217170340f9b4955443ddfda35f, State: Initialized, Role: FOLLOWER
02:06:51.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:51.088084 2207 consensus_queue.cc:260] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } }
02:06:51.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:51.092705 2211 sys_catalog.cc:455] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } } }
02:06:51.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:51.093721 2211 sys_catalog.cc:458] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [sys.catalog]: This master's current role is: FOLLOWER
02:06:51.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:51.094573 2207 sys_catalog.cc:564] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [sys.catalog]: configured and running, proceeding with master startup.
02:06:51.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:51.232378 2222 catalog_manager.cc:1261] Loaded cluster ID: 0d51723196934732bbe6c094bcb5b19d
02:06:51.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:51.232649 2222 catalog_manager.cc:1554] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f: loading cluster ID for follower catalog manager: success
02:06:51.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:51.239856 2222 catalog_manager.cc:1576] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f: acquiring CA information for follower catalog manager: success
02:06:51.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:51.245433 2222 catalog_manager.cc:1604] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:06:51.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:51 dist-test-slave-szd5 krb5kdc[863](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778810, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.254@KRBTEST.COM for kudu/127.0.212.254@KRBTEST.COM
02:06:51.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:51.317477 1342 heartbeater.cc:344] Connected to a master server at 127.0.212.254:46217
02:06:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:51.336156 1647 heartbeater.cc:344] Connected to a master server at 127.0.212.254:46217
02:06:51.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:51.344234 2172 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "ce46aaca229a4a5a9e62b9338cac9b42" instance_seqno: 1756778786352500) as {username='kudu', principal='kudu/127.0.212.193@KRBTEST.COM'} at 127.0.212.193:37127; Asking this server to re-register.
02:06:51.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:51.345384 2171 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "8f26756046ee4fb0a2e55f50a20e45ab" instance_seqno: 1756778794891758) as {username='kudu', principal='kudu/127.0.212.195@KRBTEST.COM'} at 127.0.212.195:35485; Asking this server to re-register.
02:06:51.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:51.347137 1647 heartbeater.cc:461] Registering TS with master...
02:06:51.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:51.347846 1342 heartbeater.cc:461] Registering TS with master...
02:06:51.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:51.297855 2105 master_runner.cc:186] Error getting master registration for 127.0.212.253:32829: Network error: Client connection negotiation failed: client connection to 127.0.212.253:32829: connect: Connection refused (error 111),
02:06:51.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:51.350476 2105 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:06:51.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:51.350703 2171 ts_manager.cc:194] Registered new tserver with Master: 8f26756046ee4fb0a2e55f50a20e45ab (127.0.212.195:38071)
02:06:51.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:51.352223 2170 ts_manager.cc:194] Registered new tserver with Master: ce46aaca229a4a5a9e62b9338cac9b42 (127.0.212.193:44097)
02:06:51.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:51.531358 2209 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:06:51.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:51.532155 2209 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:06:51.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:51.533242 2209 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:06:51.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:51.590478 2209 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:06:51.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:51.590817 2209 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:06:51.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:51.591297 2209 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:06:51.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:51.591662 2209 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:06:51.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:51.591985 2209 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:06:51.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:51.592293 2209 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:06:51.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:51.592526 2209 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38193/jwks.json
02:06:51.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:51.660705 2209 master_runner.cc:386] Master server non-default flags:
02:06:51.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:51.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.212.212:38213
02:06:51.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:51.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:51.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/data
02:06:51.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/wal
02:06:51.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:06:51.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:51.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.212.254:46217,127.0.212.253:32829,127.0.212.252:41741
02:06:51.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab
02:06:51.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.212.253
02:06:51.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:51.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:51.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:51.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.212.253:32829
02:06:51.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:51.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:06:51.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:51.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38193/jwks.json
02:06:51.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:51.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:06:51.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:06:51.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:51.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:51.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/data/info.pb
02:06:51.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.212.253
02:06:51.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39491
02:06:51.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:06:51.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:51.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:06:51.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:06:51.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:51.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/logs
02:06:51.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:51.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:06:51.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:06:51.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:06:51.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:06:51.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
02:06:51.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:06:51.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 01:43:19 UTC on 5fd53c4cbb9d
02:06:51.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7857
02:06:51.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:06:51.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:51.662396 2209 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:06:51.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:51.663767 2209 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:06:51.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:51.677114 2235 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:51.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:51.771613 1500 heartbeater.cc:344] Connected to a master server at 127.0.212.254:46217
02:06:51.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:51.774575 2170 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "4ac25f02a47d4bc2a47eb802eb602ee9" instance_seqno: 1756778791257506) as {username='kudu', principal='kudu/127.0.212.194@KRBTEST.COM'} at 127.0.212.194:54747; Asking this server to re-register.
02:06:51.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:51.775902 1500 heartbeater.cc:461] Registering TS with master...
02:06:51.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:51.779433 2170 ts_manager.cc:194] Registered new tserver with Master: 4ac25f02a47d4bc2a47eb802eb602ee9 (127.0.212.194:39417)
02:06:52.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:52.116350 2243 raft_consensus.cc:491] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:06:52.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:52.116954 2243 raft_consensus.cc:513] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } }
02:06:52.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:52.131196 2243 leader_election.cc:290] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers d068f217170340f9b4955443ddfda35f (127.0.212.254:46217), ced15ce4e97b4e92baa9efefcecd6a36 (127.0.212.253:32829)
02:06:52.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:52.150952 2182 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "bdab70b6529844f5a3f4d6f008ba1079" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "d068f217170340f9b4955443ddfda35f" is_pre_election: true
02:06:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:52.152256 2182 raft_consensus.cc:2466] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate bdab70b6529844f5a3f4d6f008ba1079 in term 2.
02:06:52.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:52.154192 2053 leader_election.cc:304] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: bdab70b6529844f5a3f4d6f008ba1079, d068f217170340f9b4955443ddfda35f; no voters:
02:06:52.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:52.155321 2243 raft_consensus.cc:2802] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 2 FOLLOWER]: Leader pre-election won for term 3
02:06:52.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:52.155797 2243 raft_consensus.cc:491] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:06:52.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:52.156196 2243 raft_consensus.cc:3058] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 2 FOLLOWER]: Advancing to term 3
02:06:52.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:52.177228 2055 leader_election.cc:336] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer ced15ce4e97b4e92baa9efefcecd6a36 (127.0.212.253:32829): Network error: Client connection negotiation failed: client connection to 127.0.212.253:32829: connect: Connection refused (error 111)
02:06:52.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:52.189983 2243 raft_consensus.cc:513] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } }
02:06:52.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:52.198010 2182 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "bdab70b6529844f5a3f4d6f008ba1079" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "d068f217170340f9b4955443ddfda35f"
02:06:52.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:52.198776 2182 raft_consensus.cc:3058] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [term 2 FOLLOWER]: Advancing to term 3
02:06:52.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:52.214295 2055 leader_election.cc:336] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer ced15ce4e97b4e92baa9efefcecd6a36 (127.0.212.253:32829): Network error: Client connection negotiation failed: client connection to 127.0.212.253:32829: connect: Connection refused (error 111)
02:06:52.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:52.215354 2243 leader_election.cc:290] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [CANDIDATE]: Term 3 election: Requested vote from peers d068f217170340f9b4955443ddfda35f (127.0.212.254:46217), ced15ce4e97b4e92baa9efefcecd6a36 (127.0.212.253:32829)
02:06:52.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:52.230787 2182 raft_consensus.cc:2466] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate bdab70b6529844f5a3f4d6f008ba1079 in term 3.
02:06:52.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:52.232780 2053 leader_election.cc:304] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: bdab70b6529844f5a3f4d6f008ba1079, d068f217170340f9b4955443ddfda35f; no voters: ced15ce4e97b4e92baa9efefcecd6a36
02:06:52.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:52.236282 2243 raft_consensus.cc:2802] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 3 FOLLOWER]: Leader election won for term 3
02:06:52.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:52.247726 2243 raft_consensus.cc:695] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 3 LEADER]: Becoming Leader. State: Replica: bdab70b6529844f5a3f4d6f008ba1079, State: Running, Role: LEADER
02:06:52.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:52.249130 2243 consensus_queue.cc:237] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } }
02:06:52.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:52.269692 2247 sys_catalog.cc:455] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [sys.catalog]: SysCatalogTable state changed. Reason: New leader bdab70b6529844f5a3f4d6f008ba1079. Latest consensus state: current_term: 3 leader_uuid: "bdab70b6529844f5a3f4d6f008ba1079" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } } }
02:06:52.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:52.270577 2247 sys_catalog.cc:458] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [sys.catalog]: This master's current role is: LEADER
02:06:52.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:52.300014 2249 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
02:06:52.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:52.311786 2249 catalog_manager.cc:1486] Initializing Kudu cluster ID...
02:06:52.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:52.317167 2249 catalog_manager.cc:1261] Loaded cluster ID: 0d51723196934732bbe6c094bcb5b19d
02:06:52.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:52.317718 2249 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
02:06:52.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:52.325525 2249 catalog_manager.cc:1506] Loading token signing keys...
02:06:52.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:52.331374 2249 catalog_manager.cc:5966] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079: Loaded TSK: 0
02:06:52.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:52.333093 2249 catalog_manager.cc:1516] Initializing in-progress tserver states...
02:06:52.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:52.687347 2182 raft_consensus.cc:1273] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [term 3 FOLLOWER]: Refusing update from remote peer bdab70b6529844f5a3f4d6f008ba1079: Log matching property violated. Preceding OpId in replica: term: 2 index: 5. Preceding OpId from leader: term: 3 index: 6. (index mismatch)
02:06:52.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:52.687507 2055 consensus_peers.cc:489] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 -> Peer ced15ce4e97b4e92baa9efefcecd6a36 (127.0.212.253:32829): Couldn't send request to peer ced15ce4e97b4e92baa9efefcecd6a36. Status: Network error: Client connection negotiation failed: client connection to 127.0.212.253:32829: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
02:06:52.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:52.689493 2247 consensus_queue.cc:1035] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.001s
02:06:52.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:52.732882 2252 sys_catalog.cc:455] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [sys.catalog]: SysCatalogTable state changed. Reason: New leader bdab70b6529844f5a3f4d6f008ba1079. Latest consensus state: current_term: 3 leader_uuid: "bdab70b6529844f5a3f4d6f008ba1079" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } } }
02:06:52.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:52.736146 2252 sys_catalog.cc:458] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [sys.catalog]: This master's current role is: FOLLOWER
02:06:52.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:52.740617 2252 sys_catalog.cc:455] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "bdab70b6529844f5a3f4d6f008ba1079" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } } }
02:06:52.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:52.741700 2252 sys_catalog.cc:458] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [sys.catalog]: This master's current role is: FOLLOWER
02:06:52.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:52.747973 2247 sys_catalog.cc:455] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "bdab70b6529844f5a3f4d6f008ba1079" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } } }
02:06:52.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:52.748989 2247 sys_catalog.cc:458] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [sys.catalog]: This master's current role is: LEADER
02:06:53.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:53.353675 1643 heartbeater.cc:499] Master 127.0.212.252:41741 was elected leader, sending a full tablet report...
02:06:53.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:53.673817 2018 master_runner.cc:186] Error getting master registration for 127.0.212.253:32829: Network error: Client connection negotiation failed: client connection to 127.0.212.253:32829: connect: Connection refused (error 111),
02:06:53.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:53.674441 2018 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:06:53.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:53.755671 1494 heartbeater.cc:499] Master 127.0.212.252:41741 was elected leader, sending a full tablet report...
02:06:54.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:54.277987 1338 heartbeater.cc:499] Master 127.0.212.252:41741 was elected leader, sending a full tablet report...
02:06:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:54 dist-test-slave-szd5 krb5kdc[863](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778814, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.195@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:54.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:54.499423 1526 init.cc:295] Successfully reacquired a new kerberos TGT
02:06:54.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:51.680420 2236 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:54.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:53.078836 2234 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 2209
02:06:54.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:54.612646 2234 kernel_stack_watchdog.cc:198] Thread 2209 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 395ms:
02:06:54.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:06:54.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:06:54.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:06:54.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:06:54.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:54.618327 2237 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2931 milliseconds
02:06:54.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:54.618563 2209 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.937s user 0.000s sys 0.002s
02:06:54.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:54.618993 2209 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.938s user 0.000s sys 0.002s
02:06:54.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:54.620621 2239 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:54.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:54.620625 2209 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:06:54.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:54.621917 2209 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:06:54.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:54.624267 2209 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:06:54.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:54.627779 2209 hybrid_clock.cc:648] HybridClock initialized: now 1756778814627723 us; error 59 us; skew 500 ppm
02:06:54.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:54.628546 2209 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:06:54.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:54 dist-test-slave-szd5 krb5kdc[863](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778814, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:54.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:54.634258 2209 init.cc:377] Logged in from keytab as kudu/127.0.212.253@KRBTEST.COM (short username kudu)
02:06:54.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:54.639245 2209 webserver.cc:480] Webserver started at http://127.0.212.253:39491/ using document root <none> and password file <none>
02:06:54.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:54.640194 2209 fs_manager.cc:362] Metadata directory not provided
02:06:54.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:54.640414 2209 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:06:54.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:54.648257 2209 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.007s sys 0.000s
02:06:54.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:54.652949 2271 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:06:54.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:54.653914 2209 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
02:06:54.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:54.654278 2209 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/data,/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/wal
02:06:54.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ced15ce4e97b4e92baa9efefcecd6a36"
02:06:54.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 02:06:17 on dist-test-slave-szd5"
02:06:54.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:54.656190 2209 fs_report.cc:389] FS layout report
02:06:54.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:06:54.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/wal
02:06:54.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/wal
02:06:54.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/data/data
02:06:54.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:06:54.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:06:54.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:06:54.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:06:54.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:06:54.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:06:54.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:06:54.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:06:54.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:06:54.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:06:54.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:54.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:54.711141 2209 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:06:54.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:54.718875 2209 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:54.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:54.719272 2209 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:54.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:54.792032 2209 rpc_server.cc:307] RPC server started. Bound to: 127.0.212.253:32829
02:06:54.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:54.792063 2324 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.212.253:32829 every 8 connection(s)
02:06:54.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:54.795753 2209 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/data/info.pb
02:06:54.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:54.805851 851 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu as pid 2209
02:06:54.806 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:06:54.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:54.807042 2325 sys_catalog.cc:263] Verifying existing consensus state
02:06:54.807 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.0.212.252:41741(127.0.212.252:41741)] connecting to peer
02:06:54.811 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.0.212.252:41741(127.0.212.252:41741)] Successfully connected to peer
02:06:54.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:54.813354 2325 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36: Bootstrap starting.
02:06:54.816 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.0.212.252
02:06:54.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:54 dist-test-slave-szd5 krb5kdc[863](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778805, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.0.212.252@KRBTEST.COM
02:06:54.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:54 dist-test-slave-szd5 krb5kdc[863](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778810, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.194@KRBTEST.COM for kudu/127.0.212.253@KRBTEST.COM
02:06:54.874 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xa3ab9adc, L:/127.0.0.1:60282 - R:/127.0.212.252:41741] using SASL/GSSAPI
02:06:54.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:54.888892 1495 heartbeater.cc:344] Connected to a master server at 127.0.212.253:32829
02:06:54.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:54.890525 2325 log.cc:826] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36: Log is configured to *not* fsync() on all Append() calls
02:06:54.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:54.912421 2325 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:06:54.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:54.913441 2325 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36: Bootstrap complete.
02:06:54.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:54.932824 2325 raft_consensus.cc:357] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } }
02:06:54.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:54.935634 2325 raft_consensus.cc:738] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: ced15ce4e97b4e92baa9efefcecd6a36, State: Initialized, Role: FOLLOWER
02:06:54.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:54.936510 2325 consensus_queue.cc:260] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } }
02:06:54.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:54.938865 2331 sys_catalog.cc:455] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } } }
02:06:54.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:54.939414 2331 sys_catalog.cc:458] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36 [sys.catalog]: This master's current role is: FOLLOWER
02:06:54.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:54.940280 2325 sys_catalog.cc:564] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36 [sys.catalog]: configured and running, proceeding with master startup.
02:06:54.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:54.977687 2342 catalog_manager.cc:1261] Loaded cluster ID: 0d51723196934732bbe6c094bcb5b19d
02:06:54.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:54.978029 2342 catalog_manager.cc:1554] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36: loading cluster ID for follower catalog manager: success
02:06:54.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:54.988842 2342 catalog_manager.cc:1576] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36: acquiring CA information for follower catalog manager: success
02:06:54.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:54.997179 2342 catalog_manager.cc:1604] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:06:55.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:54 dist-test-slave-szd5 krb5kdc[863](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778814, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.253@KRBTEST.COM for kudu/127.0.212.254@KRBTEST.COM
02:06:55.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:55.173944 2300 raft_consensus.cc:3058] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36 [term 2 FOLLOWER]: Advancing to term 3
02:06:55.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:55.204211 2348 sys_catalog.cc:455] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "bdab70b6529844f5a3f4d6f008ba1079" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } } }
02:06:55.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:55.205646 2348 sys_catalog.cc:458] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36 [sys.catalog]: This master's current role is: FOLLOWER
02:06:55.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:55.210376 2331 sys_catalog.cc:455] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36 [sys.catalog]: SysCatalogTable state changed. Reason: New leader bdab70b6529844f5a3f4d6f008ba1079. Latest consensus state: current_term: 3 leader_uuid: "bdab70b6529844f5a3f4d6f008ba1079" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } } }
02:06:55.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:55.211285 2331 sys_catalog.cc:458] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36 [sys.catalog]: This master's current role is: FOLLOWER
02:06:55.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:55 dist-test-slave-szd5 krb5kdc[863](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778815, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.193@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:55.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:55.277969 1217 init.cc:295] Successfully reacquired a new kerberos TGT
02:06:55.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:55.397469 1339 heartbeater.cc:344] Connected to a master server at 127.0.212.253:32829
02:06:55.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:55.403506 2290 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "ce46aaca229a4a5a9e62b9338cac9b42" instance_seqno: 1756778786352500) as {username='kudu', principal='kudu/127.0.212.193@KRBTEST.COM'} at 127.0.212.193:36615; Asking this server to re-register.
02:06:55.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:55.405112 1339 heartbeater.cc:461] Registering TS with master...
02:06:55.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:55.408306 2290 ts_manager.cc:194] Registered new tserver with Master: ce46aaca229a4a5a9e62b9338cac9b42 (127.0.212.193:44097)
02:06:55.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:55.488765 1644 heartbeater.cc:344] Connected to a master server at 127.0.212.253:32829
02:06:55.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:55.491886 2290 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "8f26756046ee4fb0a2e55f50a20e45ab" instance_seqno: 1756778794891758) as {username='kudu', principal='kudu/127.0.212.195@KRBTEST.COM'} at 127.0.212.195:42405; Asking this server to re-register.
02:06:55.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:55.493587 1644 heartbeater.cc:461] Registering TS with master...
02:06:55.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:55.496719 2290 ts_manager.cc:194] Registered new tserver with Master: 8f26756046ee4fb0a2e55f50a20e45ab (127.0.212.195:38071)
02:06:55.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:55.743779 2243 sys_catalog.cc:455] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "bdab70b6529844f5a3f4d6f008ba1079" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } } }
02:06:55.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:55.744562 2243 sys_catalog.cc:458] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [sys.catalog]: This master's current role is: LEADER
02:06:55.879 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
02:06:55.879 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
02:06:55.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:55.895968 2290 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "4ac25f02a47d4bc2a47eb802eb602ee9" instance_seqno: 1756778791257506) as {username='kudu', principal='kudu/127.0.212.194@KRBTEST.COM'} at 127.0.212.194:59405; Asking this server to re-register.
02:06:55.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:55.898250 1495 heartbeater.cc:461] Registering TS with master...
02:06:55.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:55.902477 2290 ts_manager.cc:194] Registered new tserver with Master: 4ac25f02a47d4bc2a47eb802eb602ee9 (127.0.212.194:39417)
02:06:55.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:55 dist-test-slave-szd5 krb5kdc[863](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778815, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:55.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
02:06:55.930 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:06:55.930 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/krb5cc
02:06:55.931 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:06:55.932 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.0.212.252:41741
02:06:55.932 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
02:06:55.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:55.932533 851 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu with pid 2018
02:06:55.975 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.0.212.252:41741(127.0.212.252:41741)] handling channelInactive
02:06:55.976 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.0.212.252:41741(127.0.212.252:41741)] cleaning up while in state READY due to: connection closed
02:06:55.977 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:06:55.977 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.0.212.254:46217
02:06:55.977 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
02:06:55.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:55.978060 851 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu with pid 2105
02:06:56.021 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:06:56.021 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.0.212.253:32829
02:06:56.022 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
02:06:56.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:56.022500 851 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu with pid 2209
02:06:56.051 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:06:56.051 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.0.212.252:41741
02:06:56.052 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
02:06:56.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:56.053223 851 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu
02:06:56.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu
02:06:56.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/wal
02:06:56.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/data
02:06:56.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:06:56.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:06:56.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:06:56.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:56.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:56.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:56.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/logs
02:06:56.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/data/info.pb
02:06:56.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:56.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:06:56.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:06:56.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:06:56.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:06:56.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:56.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:56.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:56.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:06:56.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:06:56.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:56.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:56.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.212.252:41741
02:06:56.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.212.252
02:06:56.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33899
02:06:56.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab
02:06:56.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.212.252
02:06:56.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:56.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:06:56.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:06:56.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:06:56.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.212.212:38213
02:06:56.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:56.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:56.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:56.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:56.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.212.254:46217,127.0.212.253:32829,127.0.212.252:41741
02:06:56.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:56.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38193/jwks.json
02:06:56.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:06:56.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:06:56.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:56.371907 2358 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:06:56.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:56.372656 2358 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:06:56.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:56.373788 2358 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:06:56.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:56.408532 2358 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:06:56.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:56.408852 2358 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:06:56.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:56.409315 2358 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:06:56.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:56.409703 2358 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:06:56.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:56.410007 2358 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:06:56.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:56.410308 2358 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:06:56.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:56.410517 2358 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38193/jwks.json
02:06:56.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:56.453079 2358 master_runner.cc:386] Master server non-default flags:
02:06:56.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:56.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.212.212:38213
02:06:56.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:56.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:56.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/data
02:06:56.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/wal
02:06:56.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:06:56.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:56.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.212.254:46217,127.0.212.253:32829,127.0.212.252:41741
02:06:56.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab
02:06:56.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.212.252
02:06:56.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:56.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:56.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:56.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.212.252:41741
02:06:56.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:56.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:06:56.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:56.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38193/jwks.json
02:06:56.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:56.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:06:56.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:06:56.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:56.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:56.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/data/info.pb
02:06:56.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.212.252
02:06:56.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33899
02:06:56.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:06:56.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:56.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:06:56.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:06:56.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:56.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/logs
02:06:56.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:56.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:06:56.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:06:56.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:06:56.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:06:56.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
02:06:56.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:06:56.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 01:43:19 UTC on 5fd53c4cbb9d
02:06:56.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7857
02:06:56.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:06:56.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:56.454367 2358 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:06:56.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:56.455348 2358 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:06:56.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:56.466140 2368 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:58.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:56.467506 2369 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:58.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:57.870200 2367 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 2358
02:06:58.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:58.189760 2358 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.723s user 0.558s sys 1.058s
02:06:58.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:58.192158 2358 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.725s user 0.558s sys 1.059s
02:06:58.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:58.192942 2371 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:58.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:58.195897 2370 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1724 milliseconds
02:06:58.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.195976 2358 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:06:58.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.197201 2358 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:06:58.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.199664 2358 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:06:58.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.201032 2358 hybrid_clock.cc:648] HybridClock initialized: now 1756778818200986 us; error 50 us; skew 500 ppm
02:06:58.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.201869 2358 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:06:58.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:58 dist-test-slave-szd5 krb5kdc[863](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778818, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:58.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.208173 2358 init.cc:377] Logged in from keytab as kudu/127.0.212.252@KRBTEST.COM (short username kudu)
02:06:58.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.213248 2358 webserver.cc:480] Webserver started at http://127.0.212.252:33899/ using document root <none> and password file <none>
02:06:58.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.214135 2358 fs_manager.cc:362] Metadata directory not provided
02:06:58.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.214323 2358 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:06:58.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.222569 2358 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.006s sys 0.000s
02:06:58.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.227061 2384 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:06:58.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.228185 2358 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s
02:06:58.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.228479 2358 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/data,/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/wal
02:06:58.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bdab70b6529844f5a3f4d6f008ba1079"
02:06:58.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 02:06:20 on dist-test-slave-szd5"
02:06:58.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.230386 2358 fs_report.cc:389] FS layout report
02:06:58.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:06:58.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/wal
02:06:58.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/wal
02:06:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/data/data
02:06:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:06:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:06:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:06:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:06:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:06:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:06:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:06:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:06:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:06:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:06:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:58.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.281363 2358 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:06:58.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.293977 2358 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:58.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.294413 2358 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:58.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.370051 2358 rpc_server.cc:307] RPC server started. Bound to: 127.0.212.252:41741
02:06:58.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.370153 2438 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.212.252:41741 every 8 connection(s)
02:06:58.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.372900 2358 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-2/data/info.pb
02:06:58.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.376600 851 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu as pid 2358
02:06:58.377 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:06:58.377 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.0.212.254:46217
02:06:58.377 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
02:06:58.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.379318 851 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu
02:06:58.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu
02:06:58.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/wal
02:06:58.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/data
02:06:58.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:06:58.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:06:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:06:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/logs
02:06:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/data/info.pb
02:06:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:06:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:06:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:06:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:06:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:06:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:06:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.212.254:46217
02:06:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.212.254
02:06:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44809
02:06:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab
02:06:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.212.254
02:06:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:06:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:06:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:06:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.212.212:38213
02:06:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:58.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:58.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.212.254:46217,127.0.212.253:32829,127.0.212.252:41741
02:06:58.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:58.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38193/jwks.json
02:06:58.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:06:58.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:06:58.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.385973 2439 sys_catalog.cc:263] Verifying existing consensus state
02:06:58.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.393083 2439 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079: Bootstrap starting.
02:06:58.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:06:58 dist-test-slave-szd5 krb5kdc[863](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778814, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.195@KRBTEST.COM for kudu/127.0.212.252@KRBTEST.COM
02:06:58.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.450403 1643 heartbeater.cc:344] Connected to a master server at 127.0.212.252:41741
02:06:58.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.457908 2439 log.cc:826] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079: Log is configured to *not* fsync() on all Append() calls
02:06:58.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.484004 2439 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:06:58.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.485121 2439 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079: Bootstrap complete.
02:06:58.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.506693 2439 raft_consensus.cc:357] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } }
02:06:58.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.509590 2439 raft_consensus.cc:738] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: bdab70b6529844f5a3f4d6f008ba1079, State: Initialized, Role: FOLLOWER
02:06:58.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.510599 2439 consensus_queue.cc:260] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } }
02:06:58.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.513906 2446 sys_catalog.cc:455] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } } }
02:06:58.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.514717 2446 sys_catalog.cc:458] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [sys.catalog]: This master's current role is: FOLLOWER
02:06:58.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.515764 2439 sys_catalog.cc:564] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [sys.catalog]: configured and running, proceeding with master startup.
02:06:58.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.550873 2457 catalog_manager.cc:1261] Loaded cluster ID: 0d51723196934732bbe6c094bcb5b19d
02:06:58.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.551337 2457 catalog_manager.cc:1554] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079: loading cluster ID for follower catalog manager: success
02:06:58.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.559504 2457 catalog_manager.cc:1576] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079: acquiring CA information for follower catalog manager: success
02:06:58.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.572038 2457 catalog_manager.cc:1604] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:06:58.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.545459 2358 master_runner.cc:186] Error getting master registration for 127.0.212.254:46217: Network error: Client connection negotiation failed: client connection to 127.0.212.254:46217: connect: Connection refused (error 111),
02:06:58.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.577040 2358 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:06:58.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:58.758481 2441 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:06:58.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:58.759123 2441 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:06:58.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:58.759974 2441 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:06:58.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:58.792949 2441 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:06:58.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:58.793231 2441 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:06:58.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:58.793637 2441 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:06:58.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:58.793941 2441 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:06:58.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:58.794252 2441 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:06:58.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:58.794556 2441 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:06:58.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:58.794780 2441 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38193/jwks.json
02:06:58.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.817395 1494 heartbeater.cc:344] Connected to a master server at 127.0.212.252:41741
02:06:58.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.825896 2403 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "4ac25f02a47d4bc2a47eb802eb602ee9" instance_seqno: 1756778791257506) as {username='kudu', principal='kudu/127.0.212.194@KRBTEST.COM'} at 127.0.212.194:33909; Asking this server to re-register.
02:06:58.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.827242 1494 heartbeater.cc:461] Registering TS with master...
02:06:58.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:58.830643 32665 debug-util.cc:398] Leaking SignalData structure 0x7b0800087be0 after lost signal to thread 32602
02:06:58.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.830626 2403 ts_manager.cc:194] Registered new tserver with Master: 4ac25f02a47d4bc2a47eb802eb602ee9 (127.0.212.194:39417)
02:06:58.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:58.831321 32665 debug-util.cc:398] Leaking SignalData structure 0x7b080009ef60 after lost signal to thread 32668
02:06:58.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.849858 2441 master_runner.cc:386] Master server non-default flags:
02:06:58.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:58.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.212.212:38213
02:06:58.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:58.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:58.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/data
02:06:58.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/wal
02:06:58.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:06:58.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:58.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.212.254:46217,127.0.212.253:32829,127.0.212.252:41741
02:06:58.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab
02:06:58.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.212.254
02:06:58.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:58.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:58.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:58.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.212.254:46217
02:06:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:06:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38193/jwks.json
02:06:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:06:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:06:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/data/info.pb
02:06:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.212.254
02:06:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44809
02:06:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:06:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:06:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:06:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/logs
02:06:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:06:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:06:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:06:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:06:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
02:06:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:06:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 01:43:19 UTC on 5fd53c4cbb9d
02:06:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7857
02:06:58.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:06:58.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.851737 2441 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:06:58.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:58.853283 2441 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:06:58.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:58.869103 2467 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:59.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:59.117619 1490 debug-util.cc:398] Leaking SignalData structure 0x7b08000bade0 after lost signal to thread 1361
02:06:59.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:59.119684 1490 debug-util.cc:398] Leaking SignalData structure 0x7b08000aee00 after lost signal to thread 1493
02:06:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:59.338822 1338 heartbeater.cc:344] Connected to a master server at 127.0.212.252:41741
02:06:59.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:59.342070 2403 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "ce46aaca229a4a5a9e62b9338cac9b42" instance_seqno: 1756778786352500) as {username='kudu', principal='kudu/127.0.212.193@KRBTEST.COM'} at 127.0.212.193:53911; Asking this server to re-register.
02:06:59.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:59.343976 1338 heartbeater.cc:461] Registering TS with master...
02:06:59.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:59.347498 2403 ts_manager.cc:194] Registered new tserver with Master: ce46aaca229a4a5a9e62b9338cac9b42 (127.0.212.193:44097)
02:06:59.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:59.466666 2403 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "8f26756046ee4fb0a2e55f50a20e45ab" instance_seqno: 1756778794891758) as {username='kudu', principal='kudu/127.0.212.195@KRBTEST.COM'} at 127.0.212.195:34679; Asking this server to re-register.
02:06:59.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:59.471043 1643 heartbeater.cc:461] Registering TS with master...
02:06:59.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:06:59.475157 2403 ts_manager.cc:194] Registered new tserver with Master: 8f26756046ee4fb0a2e55f50a20e45ab (127.0.212.195:38071)
02:07:00.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:00.065130 2475 raft_consensus.cc:491] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:07:00.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:00.066696 2475 raft_consensus.cc:513] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } }
02:07:00.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:00.104961 2475 leader_election.cc:290] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers d068f217170340f9b4955443ddfda35f (127.0.212.254:46217), ced15ce4e97b4e92baa9efefcecd6a36 (127.0.212.253:32829)
02:07:00.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:07:00.137032 2389 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.212.253:32829: connect: Connection refused (error 111)
02:07:00.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:07:00.167431 2387 leader_election.cc:336] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer d068f217170340f9b4955443ddfda35f (127.0.212.254:46217): Network error: Client connection negotiation failed: client connection to 127.0.212.254:46217: connect: Connection refused (error 111)
02:07:00.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:07:00.173959 2389 leader_election.cc:336] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer ced15ce4e97b4e92baa9efefcecd6a36 (127.0.212.253:32829): Network error: Client connection negotiation failed: client connection to 127.0.212.253:32829: connect: Connection refused (error 111)
02:07:00.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:00.190644 2389 leader_election.cc:304] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: bdab70b6529844f5a3f4d6f008ba1079; no voters: ced15ce4e97b4e92baa9efefcecd6a36, d068f217170340f9b4955443ddfda35f
02:07:00.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:00.196427 2475 raft_consensus.cc:2747] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
02:07:00.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:07:00.769555 833 debug-util.cc:398] Leaking SignalData structure 0x7b08000b6d80 after lost signal to thread 707
02:07:00.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:07:00.770753 833 debug-util.cc:398] Leaking SignalData structure 0x7b08000b6f40 after lost signal to thread 836
02:07:00.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:07:00 dist-test-slave-szd5 krb5kdc[863](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778820, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.194@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:07:00.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:00.847676 1376 init.cc:295] Successfully reacquired a new kerberos TGT
02:07:00.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:07:00.269316 2466 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 2441
02:07:00.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:07:00.987744 2441 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.121s user 0.634s sys 1.123s
02:07:00.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:07:00.991969 2441 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.125s user 0.635s sys 1.123s
02:07:00.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:06:58.870824 2468 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:07:01.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:01.014883 2441 server_base.cc:1047] running on GCE node
02:07:01.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:07:01.017279 2485 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:07:01.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:01.019075 2441 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:07:01.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:01.027341 2441 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:07:01.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:01.031252 2441 hybrid_clock.cc:648] HybridClock initialized: now 1756778821031131 us; error 117 us; skew 500 ppm
02:07:01.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:01.032811 2441 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:07:01.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:07:01 dist-test-slave-szd5 krb5kdc[863](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778821, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:07:01.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:01.049520 2441 init.cc:377] Logged in from keytab as kudu/127.0.212.254@KRBTEST.COM (short username kudu)
02:07:01.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:01.085310 2441 webserver.cc:480] Webserver started at http://127.0.212.254:44809/ using document root <none> and password file <none>
02:07:01.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:01.086659 2441 fs_manager.cc:362] Metadata directory not provided
02:07:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:01.086962 2441 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:07:01.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:01.099228 2441 fs_manager.cc:714] Time spent opening directory manager: real 0.008s user 0.004s sys 0.000s
02:07:01.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:01.116536 2491 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:07:01.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:01.119091 2441 fs_manager.cc:730] Time spent opening block manager: real 0.016s user 0.002s sys 0.011s
02:07:01.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:01.119812 2441 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/data,/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/wal
02:07:01.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d068f217170340f9b4955443ddfda35f"
02:07:01.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 02:06:14 on dist-test-slave-szd5"
02:07:01.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:01.123176 2441 fs_report.cc:389] FS layout report
02:07:01.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:07:01.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/wal
02:07:01.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/wal
02:07:01.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/data/data
02:07:01.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:07:01.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:07:01.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:07:01.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:07:01.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:07:01.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:07:01.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:07:01.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:07:01.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:07:01.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:07:01.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:07:01.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:01.265913 2441 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:07:01.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:01.275420 2441 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:07:01.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:01.276381 2441 kserver.cc:163] Server-wide thread pool size limit: 3276
02:07:01.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:01.722224 2545 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.212.254:46217 every 8 connection(s)
02:07:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:01.723178 2441 rpc_server.cc:307] RPC server started. Bound to: 127.0.212.254:46217
02:07:01.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:01.728139 2441 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-0/data/info.pb
02:07:01.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:01.730504 851 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu as pid 2441
02:07:01.731 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:07:01.731 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.0.212.253:32829
02:07:01.732 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
02:07:01.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:01.734169 851 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu
02:07:01.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu
02:07:01.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/wal
02:07:01.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/data
02:07:01.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:07:01.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:07:01.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:07:01.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:07:01.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:07:01.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:07:01.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/logs
02:07:01.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/data/info.pb
02:07:01.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:07:01.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:07:01.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:07:01.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:07:01.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:07:01.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:07:01.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:07:01.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:07:01.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:07:01.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:07:01.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:07:01.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:07:01.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.212.253:32829
02:07:01.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.212.253
02:07:01.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39491
02:07:01.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab
02:07:01.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.212.253
02:07:01.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:07:01.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:07:01.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:07:01.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:07:01.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.212.212:38213
02:07:01.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:07:01.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:07:01.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:07:01.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:07:01.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.212.254:46217,127.0.212.253:32829,127.0.212.252:41741
02:07:01.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:07:01.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38193/jwks.json
02:07:01.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:07:01.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:07:01.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:01.770774 2546 sys_catalog.cc:263] Verifying existing consensus state
02:07:01.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:01.792575 2546 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f: Bootstrap starting.
02:07:01.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:01.871560 2546 log.cc:826] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f: Log is configured to *not* fsync() on all Append() calls
02:07:01.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:01.906896 2546 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:07:01.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:01.907931 2546 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f: Bootstrap complete.
02:07:01.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:07:01 dist-test-slave-szd5 krb5kdc[863](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778820, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.194@KRBTEST.COM for kudu/127.0.212.254@KRBTEST.COM
02:07:01.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:01.970530 2546 raft_consensus.cc:357] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } }
02:07:01.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:01.974112 2546 raft_consensus.cc:738] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: d068f217170340f9b4955443ddfda35f, State: Initialized, Role: FOLLOWER
02:07:01.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:01.975382 2546 consensus_queue.cc:260] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } }
02:07:01.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:01.990897 2546 sys_catalog.cc:564] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [sys.catalog]: configured and running, proceeding with master startup.
02:07:01.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:01.992379 2553 sys_catalog.cc:455] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } } }
02:07:01.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:01.997285 2553 sys_catalog.cc:458] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [sys.catalog]: This master's current role is: FOLLOWER
02:07:02.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.035223 1500 heartbeater.cc:344] Connected to a master server at 127.0.212.254:46217
02:07:02.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.152032 2565 catalog_manager.cc:797] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
02:07:02.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:07:02 dist-test-slave-szd5 krb5kdc[863](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778821, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.254@KRBTEST.COM for kudu/127.0.212.254@KRBTEST.COM
02:07:02.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.196260 2567 raft_consensus.cc:491] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:07:02.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.196736 2567 raft_consensus.cc:513] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } }
02:07:02.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.225411 2567 leader_election.cc:290] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers d068f217170340f9b4955443ddfda35f (127.0.212.254:46217), ced15ce4e97b4e92baa9efefcecd6a36 (127.0.212.253:32829)
02:07:02.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:07:02.243978 2389 leader_election.cc:336] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer ced15ce4e97b4e92baa9efefcecd6a36 (127.0.212.253:32829): Network error: Client connection negotiation failed: client connection to 127.0.212.253:32829: connect: Connection refused (error 111)
02:07:02.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:07:02 dist-test-slave-szd5 krb5kdc[863](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778818, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.252@KRBTEST.COM for kudu/127.0.212.254@KRBTEST.COM
02:07:02.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.293279 2521 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "bdab70b6529844f5a3f4d6f008ba1079" candidate_term: 4 candidate_status { last_received { term: 3 index: 6 } } ignore_live_leader: false dest_uuid: "d068f217170340f9b4955443ddfda35f" is_pre_election: true
02:07:02.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.302073 2521 raft_consensus.cc:2466] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate bdab70b6529844f5a3f4d6f008ba1079 in term 3.
02:07:02.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.304023 2387 leader_election.cc:304] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: bdab70b6529844f5a3f4d6f008ba1079, d068f217170340f9b4955443ddfda35f; no voters: ced15ce4e97b4e92baa9efefcecd6a36
02:07:02.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.304927 2567 raft_consensus.cc:2802] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 3 FOLLOWER]: Leader pre-election won for term 4
02:07:02.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.305272 2567 raft_consensus.cc:491] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 3 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:07:02.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.305595 2567 raft_consensus.cc:3058] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 3 FOLLOWER]: Advancing to term 4
02:07:02.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.246117 2441 master_runner.cc:186] Error getting master registration for 127.0.212.253:32829: Network error: Client connection negotiation failed: client connection to 127.0.212.253:32829: connect: Connection refused (error 111),
02:07:02.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.306241 2441 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:07:02.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.315816 2567 raft_consensus.cc:513] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } }
02:07:02.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.318890 2521 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "bdab70b6529844f5a3f4d6f008ba1079" candidate_term: 4 candidate_status { last_received { term: 3 index: 6 } } ignore_live_leader: false dest_uuid: "d068f217170340f9b4955443ddfda35f"
02:07:02.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.319520 2521 raft_consensus.cc:3058] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [term 3 FOLLOWER]: Advancing to term 4
02:07:02.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.329814 2521 raft_consensus.cc:2466] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate bdab70b6529844f5a3f4d6f008ba1079 in term 4.
02:07:02.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.331040 2387 leader_election.cc:304] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: bdab70b6529844f5a3f4d6f008ba1079, d068f217170340f9b4955443ddfda35f; no voters:
02:07:02.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:07:02.341549 2389 leader_election.cc:336] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer ced15ce4e97b4e92baa9efefcecd6a36 (127.0.212.253:32829): Network error: Client connection negotiation failed: client connection to 127.0.212.253:32829: connect: Connection refused (error 111)
02:07:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.342669 2567 leader_election.cc:290] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [CANDIDATE]: Term 4 election: Requested vote from peers d068f217170340f9b4955443ddfda35f (127.0.212.254:46217), ced15ce4e97b4e92baa9efefcecd6a36 (127.0.212.253:32829)
02:07:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.343178 2567 raft_consensus.cc:2802] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 4 FOLLOWER]: Leader election won for term 4
02:07:02.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.344646 2567 raft_consensus.cc:695] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [term 4 LEADER]: Becoming Leader. State: Replica: bdab70b6529844f5a3f4d6f008ba1079, State: Running, Role: LEADER
02:07:02.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.345695 2567 consensus_queue.cc:237] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } }
02:07:02.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.362709 2573 sys_catalog.cc:455] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [sys.catalog]: SysCatalogTable state changed. Reason: New leader bdab70b6529844f5a3f4d6f008ba1079. Latest consensus state: current_term: 4 leader_uuid: "bdab70b6529844f5a3f4d6f008ba1079" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } } }
02:07:02.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.363806 2573 sys_catalog.cc:458] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [sys.catalog]: This master's current role is: LEADER
02:07:02.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.378839 2575 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
02:07:02.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.387512 2575 catalog_manager.cc:1486] Initializing Kudu cluster ID...
02:07:02.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.391993 2575 catalog_manager.cc:1261] Loaded cluster ID: 0d51723196934732bbe6c094bcb5b19d
02:07:02.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.392274 2575 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
02:07:02.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.397569 2575 catalog_manager.cc:1506] Loading token signing keys...
02:07:02.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.402138 2575 catalog_manager.cc:5966] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079: Loaded TSK: 0
02:07:02.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.403510 2575 catalog_manager.cc:1516] Initializing in-progress tserver states...
02:07:02.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:07:02 dist-test-slave-szd5 krb5kdc[863](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778815, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.193@KRBTEST.COM for kudu/127.0.212.254@KRBTEST.COM
02:07:02.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.489200 1342 heartbeater.cc:344] Connected to a master server at 127.0.212.254:46217
02:07:02.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.497283 2508 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "ce46aaca229a4a5a9e62b9338cac9b42" instance_seqno: 1756778786352500) as {username='kudu', principal='kudu/127.0.212.193@KRBTEST.COM'} at 127.0.212.193:34743; Asking this server to re-register.
02:07:02.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.500622 1342 heartbeater.cc:461] Registering TS with master...
02:07:02.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.504112 2507 ts_manager.cc:194] Registered new tserver with Master: ce46aaca229a4a5a9e62b9338cac9b42 (127.0.212.193:44097)
02:07:02.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:07:02 dist-test-slave-szd5 krb5kdc[863](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778814, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.195@KRBTEST.COM for kudu/127.0.212.254@KRBTEST.COM
02:07:02.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.577113 1647 heartbeater.cc:344] Connected to a master server at 127.0.212.254:46217
02:07:02.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.581547 2508 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "8f26756046ee4fb0a2e55f50a20e45ab" instance_seqno: 1756778794891758) as {username='kudu', principal='kudu/127.0.212.195@KRBTEST.COM'} at 127.0.212.195:40007; Asking this server to re-register.
02:07:02.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.589999 1647 heartbeater.cc:461] Registering TS with master...
02:07:02.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.594434 2508 ts_manager.cc:194] Registered new tserver with Master: 8f26756046ee4fb0a2e55f50a20e45ab (127.0.212.195:38071)
02:07:02.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:07:02.632094 2548 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:07:02.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:07:02.632727 2548 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:07:02.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:07:02.633445 2548 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:07:02.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:07:02.699321 2548 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:07:02.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:07:02.699710 2548 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:07:02.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:07:02.700268 2548 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:07:02.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:07:02.700645 2548 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:07:02.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:07:02.700999 2548 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:07:02.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:07:02.701335 2548 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:07:02.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:07:02.701613 2548 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38193/jwks.json
02:07:02.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:07:02.765667 2389 consensus_peers.cc:489] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 -> Peer ced15ce4e97b4e92baa9efefcecd6a36 (127.0.212.253:32829): Couldn't send request to peer ced15ce4e97b4e92baa9efefcecd6a36. Status: Network error: Client connection negotiation failed: client connection to 127.0.212.253:32829: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
02:07:02.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.769856 2548 master_runner.cc:386] Master server non-default flags:
02:07:02.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:07:02.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.212.212:38213
02:07:02.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:07:02.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:07:02.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/data
02:07:02.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/wal
02:07:02.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:07:02.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:07:02.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.212.254:46217,127.0.212.253:32829,127.0.212.252:41741
02:07:02.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/krb5kdc/kudu.keytab
02:07:02.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.212.253
02:07:02.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:07:02.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:07:02.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:07:02.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.212.253:32829
02:07:02.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:07:02.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:07:02.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:07:02.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38193/jwks.json
02:07:02.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:07:02.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:07:02.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:07:02.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:07:02.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:07:02.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/data/info.pb
02:07:02.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.212.253
02:07:02.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39491
02:07:02.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:07:02.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:07:02.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:07:02.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:07:02.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:07:02.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/logs
02:07:02.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:07:02.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:07:02.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:07:02.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:07:02.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:07:02.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
02:07:02.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:07:02.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 01:43:19 UTC on 5fd53c4cbb9d
02:07:02.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7857
02:07:02.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:07:02.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.771556 2548 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:07:02.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.772904 2548 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:07:02.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:07:02.795223 2586 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:07:02.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.903003 2521 raft_consensus.cc:1273] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [term 4 FOLLOWER]: Refusing update from remote peer bdab70b6529844f5a3f4d6f008ba1079: Log matching property violated. Preceding OpId in replica: term: 3 index: 6. Preceding OpId from leader: term: 4 index: 7. (index mismatch)
02:07:02.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.905128 2573 consensus_queue.cc:1035] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
02:07:02.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.970927 2573 sys_catalog.cc:455] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "bdab70b6529844f5a3f4d6f008ba1079" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } } }
02:07:02.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.972011 2573 sys_catalog.cc:458] T 00000000000000000000000000000000 P bdab70b6529844f5a3f4d6f008ba1079 [sys.catalog]: This master's current role is: LEADER
02:07:02.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.979872 2590 sys_catalog.cc:455] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [sys.catalog]: SysCatalogTable state changed. Reason: New leader bdab70b6529844f5a3f4d6f008ba1079. Latest consensus state: current_term: 4 leader_uuid: "bdab70b6529844f5a3f4d6f008ba1079" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } } }
02:07:02.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.987875 2590 sys_catalog.cc:458] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [sys.catalog]: This master's current role is: FOLLOWER
02:07:02.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.989269 2597 sys_catalog.cc:455] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "bdab70b6529844f5a3f4d6f008ba1079" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } } }
02:07:02.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:02.990358 2597 sys_catalog.cc:458] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f [sys.catalog]: This master's current role is: FOLLOWER
02:07:03.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:03.043994 2508 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "4ac25f02a47d4bc2a47eb802eb602ee9" instance_seqno: 1756778791257506) as {username='kudu', principal='kudu/127.0.212.194@KRBTEST.COM'} at 127.0.212.194:55491; Asking this server to re-register.
02:07:03.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:03.045989 1500 heartbeater.cc:461] Registering TS with master...
02:07:03.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:03.049911 2508 ts_manager.cc:194] Registered new tserver with Master: 4ac25f02a47d4bc2a47eb802eb602ee9 (127.0.212.194:39417)
02:07:03.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:03.177800 2565 catalog_manager.cc:1261] Loaded cluster ID: 0d51723196934732bbe6c094bcb5b19d
02:07:03.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:03.178248 2565 catalog_manager.cc:1554] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f: loading cluster ID for follower catalog manager: success
02:07:03.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:03.202622 2565 catalog_manager.cc:1576] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f: acquiring CA information for follower catalog manager: success
02:07:03.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:03.218173 2565 catalog_manager.cc:1604] T 00000000000000000000000000000000 P d068f217170340f9b4955443ddfda35f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:07:03.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:03.489813 1643 heartbeater.cc:499] Master 127.0.212.252:41741 was elected leader, sending a full tablet report...
02:07:03.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:03.587749 2358 master_runner.cc:186] Error getting master registration for 127.0.212.253:32829: Network error: Client connection negotiation failed: client connection to 127.0.212.253:32829: connect: Connection refused (error 111),
02:07:03.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:03.588250 2358 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:07:03.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:03.861079 1494 heartbeater.cc:499] Master 127.0.212.252:41741 was elected leader, sending a full tablet report...
02:07:04.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:04.378333 1338 heartbeater.cc:499] Master 127.0.212.252:41741 was elected leader, sending a full tablet report...
02:07:04.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:07:04 dist-test-slave-szd5 krb5kdc[863](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778824, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.195@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:07:04.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:04.503278 1526 init.cc:295] Successfully reacquired a new kerberos TGT
02:07:04.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:07:02.807027 2587 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:07:04.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:07:04.754607 2548 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.950s user 0.622s sys 0.899s
02:07:04.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:07:04.194785 2585 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 2548
02:07:04.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:07:04.755153 2548 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.950s user 0.622s sys 0.899s
02:07:04.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:07:04.762923 2612 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:07:04.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:04.763343 2548 server_base.cc:1047] running on GCE node
02:07:04.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:04.764532 2548 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:07:04.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:04.767848 2548 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:07:04.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:04.769609 2548 hybrid_clock.cc:648] HybridClock initialized: now 1756778824769568 us; error 43 us; skew 500 ppm
02:07:04.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:04.770771 2548 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:07:04.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:07:04 dist-test-slave-szd5 krb5kdc[863](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778824, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:07:04.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:04.777391 2548 init.cc:377] Logged in from keytab as kudu/127.0.212.253@KRBTEST.COM (short username kudu)
02:07:04.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:04.783383 2548 webserver.cc:480] Webserver started at http://127.0.212.253:39491/ using document root <none> and password file <none>
02:07:04.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:04.784314 2548 fs_manager.cc:362] Metadata directory not provided
02:07:04.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:04.784505 2548 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:07:04.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:04.793386 2548 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.000s sys 0.004s
02:07:04.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:04.798900 2619 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:07:04.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:04.800036 2548 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
02:07:04.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:04.800376 2548 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/data,/tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/wal
02:07:04.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ced15ce4e97b4e92baa9efefcecd6a36"
02:07:04.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 02:06:17 on dist-test-slave-szd5"
02:07:04.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:04.802378 2548 fs_report.cc:389] FS layout report
02:07:04.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:07:04.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/wal
02:07:04.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/wal
02:07:04.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/data/data
02:07:04.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:07:04.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:07:04.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:07:04.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:07:04.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:07:04.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:07:04.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:07:04.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:07:04.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:07:04.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:07:04.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:07:04.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:04.874859 2548 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:07:04.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:04.885389 2548 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:07:04.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:04.885809 2548 kserver.cc:163] Server-wide thread pool size limit: 3276
02:07:05.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:05.001266 2548 rpc_server.cc:307] RPC server started. Bound to: 127.0.212.253:32829
02:07:05.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:05.001367 2671 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.212.253:32829 every 8 connection(s)
02:07:05.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:05.004287 2548 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskONRpAo/test-tmp/mini-kudu-cluster2062730607172396139/master-1/data/info.pb
02:07:05.007 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:07:05.007 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.0.212.252:41741(127.0.212.252:41741)] connecting to peer
02:07:05.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:05.006404 851 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu as pid 2548
02:07:05.011 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.0.212.252:41741(127.0.212.252:41741)] Successfully connected to peer
02:07:05.013 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.212.252:41741(127.0.212.252:41741)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
02:07:05.015 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.0.212.252:41741(127.0.212.252:41741)] handling channelInactive
02:07:05.017 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.0.212.254:46217(127.0.212.254:46217)] disconnecting while in state TERMINATED
02:07:05.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:05.021268 2672 sys_catalog.cc:263] Verifying existing consensus state
02:07:05.021 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.0.212.252:41741(127.0.212.252:41741)] disconnecting while in state TERMINATED
02:07:05.022 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.0.212.253:32829(127.0.212.253:32829)] disconnecting while in state TERMINATED
02:07:05.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 02:07:05.025643 2674 negotiation.cc:337] Failed RPC negotiation. Trace:
02:07:05.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 02:07:05.010273 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:35012 (local address 127.0.212.252:41741)
02:07:05.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 02:07:05.011813 (+ 1540us) server_negotiation.cc:206] Beginning negotiation
02:07:05.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 02:07:05.011843 (+ 30us) server_negotiation.cc:399] Waiting for connection header
02:07:05.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 02:07:05.011920 (+ 77us) server_negotiation.cc:407] Connection header received
02:07:05.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 02:07:05.012303 (+ 383us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
02:07:05.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 02:07:05.012323 (+ 20us) server_negotiation.cc:461] Received NEGOTIATE request from client
02:07:05.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 02:07:05.012498 (+ 175us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
02:07:05.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 02:07:05.012663 (+ 165us) server_negotiation.cc:226] Negotiated authn=SASL
02:07:05.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 02:07:05.024197 (+ 11534us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:35012: BlockingRecv error: recv got EOF from 127.0.0.1:35012 (error 108)
02:07:05.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":932,"thread_start_us":649,"threads_started":1}
02:07:05.027 [DEBUG - Time-limited test] (AsyncKuduClient.java:2704) Releasing all remaining resources
02:07:05.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:05.029206 2672 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36: Bootstrap starting.
02:07:05.036 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
02:07:05.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:05.067437 851 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu with pid 1177
02:07:05.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:05.092203 2672 log.cc:826] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36: Log is configured to *not* fsync() on all Append() calls
02:07:05.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:05.112746 851 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu with pid 1352
02:07:05.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:05.122112 2672 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:07:05.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:05.123211 2672 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36: Bootstrap complete.
02:07:05.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:05.170444 851 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu with pid 1508
02:07:05.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:05.166325 2672 raft_consensus.cc:357] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } }
02:07:05.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:05.183766 2672 raft_consensus.cc:738] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: ced15ce4e97b4e92baa9efefcecd6a36, State: Initialized, Role: FOLLOWER
02:07:05.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:05.188519 2672 consensus_queue.cc:260] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } }
02:07:05.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 02:07:05 dist-test-slave-szd5 krb5kdc[863](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756778818, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.212.252@KRBTEST.COM for kudu/127.0.212.253@KRBTEST.COM
02:07:05.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:05.211905 851 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu with pid 2441
02:07:05.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:05.216622 2646 raft_consensus.cc:3058] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36 [term 3 FOLLOWER]: Advancing to term 4
02:07:05.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:05.211656 2681 sys_catalog.cc:455] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } } }
02:07:05.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:05.218014 2681 sys_catalog.cc:458] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36 [sys.catalog]: This master's current role is: FOLLOWER
02:07:05.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:05.221462 2672 sys_catalog.cc:564] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36 [sys.catalog]: configured and running, proceeding with master startup.
02:07:05.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:05.239038 2687 sys_catalog.cc:455] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "bdab70b6529844f5a3f4d6f008ba1079" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } } }
02:07:05.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:05.240026 2687 sys_catalog.cc:458] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36 [sys.catalog]: This master's current role is: FOLLOWER
02:07:05.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:05.243922 2681 sys_catalog.cc:455] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36 [sys.catalog]: SysCatalogTable state changed. Reason: New leader bdab70b6529844f5a3f4d6f008ba1079. Latest consensus state: current_term: 4 leader_uuid: "bdab70b6529844f5a3f4d6f008ba1079" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d068f217170340f9b4955443ddfda35f" member_type: VOTER last_known_addr { host: "127.0.212.254" port: 46217 } } peers { permanent_uuid: "ced15ce4e97b4e92baa9efefcecd6a36" member_type: VOTER last_known_addr { host: "127.0.212.253" port: 32829 } } peers { permanent_uuid: "bdab70b6529844f5a3f4d6f008ba1079" member_type: VOTER last_known_addr { host: "127.0.212.252" port: 41741 } } }
02:07:05.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:05.244748 2681 sys_catalog.cc:458] T 00000000000000000000000000000000 P ced15ce4e97b4e92baa9efefcecd6a36 [sys.catalog]: This master's current role is: FOLLOWER
02:07:05.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:05.258083 851 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu with pid 2548
02:07:05.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:05.279860 851 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu with pid 2358
02:07:06.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-02T02:07:06Z chronyd exiting
02:07:07.520 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
02:07:07.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:07.521538 32589 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu with pid 412
02:07:07.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:07.547006 32589 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu with pid 558
02:07:07.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:07.574587 32589 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu with pid 705
02:07:07.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:07.604367 32589 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu with pid 32601
02:07:07.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:07.635843 32589 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu with pid 32671
02:07:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 02:07:07.668124 32589 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskONRpAo/build/tsan/bin/kudu with pid 32746
02:07:08.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-02T02:07:08Z chronyd exiting
02:07:09.848 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565) ~[main/:?]
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
02:07:10.300 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
02:07:10.300 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565) ~[main/:?]
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more