20:00:44.108 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
20:00:44.118 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task_muOGM/test-tmp
20:00:44.123 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
20:00:44.126 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task_muOGM/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
20:00:44.330 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
20:00:44.509 [DEBUG - main] (MiniKuduCluster.java:195) Response:
20:00:44.510 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
20:00:44.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:44.514955 15305 mini_oidc.cc:150] Starting JWKS server
20:00:44.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:44.518461 15305 webserver.cc:533] Webserver started at http://127.0.0.1:35507/ using document root <none> and password file <none>
20:00:44.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:44.518908 15305 mini_oidc.cc:186] Starting OIDC Discovery server
20:00:44.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:44.520960 15305 webserver.cc:533] Webserver started at http://127.0.0.1:39525/ using document root <none> and password file <none>
20:00:44.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T20:00:44Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
20:00:44.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T20:00:44Z Disabled control of system clock
20:00:44.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:44.583702 15305 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_muOGM/build/tsan/bin/kudu
20:00:44.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_muOGM/build/tsan/bin/kudu
20:00:44.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-0/wal
20:00:44.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-0/data
20:00:44.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:00:44.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:00:44.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:00:44.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:00:44.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:00:44.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:00:44.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-0/logs
20:00:44.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-0/data/info.pb
20:00:44.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:00:44.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:00:44.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:00:44.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:00:44.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:00:44.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:00:44.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:00:44.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:00:44.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:00:44.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:00:44.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:00:44.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:00:44.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.242.126:44505
20:00:44.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.242.126
20:00:44.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:00:44.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.242.84:46193
20:00:44.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:00:44.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:00:44.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:00:44.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:00:44.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.242.126:44505,127.14.242.125:42633,127.14.242.124:32797
20:00:44.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:00:44.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35507/jwks.json with env {}
20:00:44.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:44.939306 15317 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:00:44.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:44.940258 15317 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:00:44.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:44.941023 15317 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:00:44.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:44.977576 15317 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:00:44.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:44.977876 15317 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:00:44.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:44.978143 15317 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:00:44.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:44.978497 15317 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:00:44.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:44.978825 15317 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:00:44.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:44.979084 15317 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:35507/jwks.json
20:00:45.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:45.016366 15317 master_runner.cc:387] Master server non-default flags:
20:00:45.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-0/logs
20:00:45.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:00:45.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:00:45.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:00:45.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.242.84:46193
20:00:45.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:00:45.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:00:45.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-0/data
20:00:45.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-0/wal
20:00:45.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:00:45.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.242.126:44505,127.14.242.125:42633,127.14.242.124:32797
20:00:45.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:00:45.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:00:45.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:00:45.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.242.126:44505
20:00:45.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:00:45.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:00:45.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:00:45.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35507/jwks.json
20:00:45.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:00:45.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:00:45.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-0/data/info.pb
20:00:45.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.242.126
20:00:45.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:00:45.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:00:45.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:00:45.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:00:45.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:00:45.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:00:45.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:00:45.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:00:45.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:00:45.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3b18f0139b1d21a072b7353269301fba16dca54e
20:00:45.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:00:45.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Sep 2026 19:43:26 UTC on e7f111948823
20:00:45.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14030
20:00:45.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:00:45.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:45.017751 15317 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:00:45.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:45.020020 15317 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:00:45.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:45.034473 15323 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:46.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:45.036890 15324 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:46.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:45.043830 15317 server_base.cc:1061] running on GCE node
20:00:46.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:46.253744 15326 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:46.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:46.255882 15317 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:00:46.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:46.258548 15317 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:00:46.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:46.259938 15317 hybrid_clock.cc:648] HybridClock initialized: now 1790539246259873 us; error 53 us; skew 500 ppm
20:00:46.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:46.260957 15317 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:00:46.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:46.268448 15317 webserver.cc:533] Webserver started at http://127.14.242.126:44971/ using document root <none> and password file <none>
20:00:46.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:46.269533 15317 fs_manager.cc:362] Metadata directory not provided
20:00:46.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:46.269793 15317 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:00:46.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:46.270412 15317 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:00:46.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:46.274856 15317 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-0/data/instance:
20:00:46.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6b00bb10f6da46b8ab5a69d8904a4fc5"
20:00:46.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:00:46 on dist-test-slave-1555"
20:00:46.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:46.276358 15317 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-0/wal/instance:
20:00:46.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6b00bb10f6da46b8ab5a69d8904a4fc5"
20:00:46.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:00:46 on dist-test-slave-1555"
20:00:46.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:46.283917 15317 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.002s sys 0.005s
20:00:46.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:46.290817 15333 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:00:46.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:46.292090 15317 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.003s sys 0.004s
20:00:46.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:46.292469 15317 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-0/data,/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-0/wal
20:00:46.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6b00bb10f6da46b8ab5a69d8904a4fc5"
20:00:46.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:00:46 on dist-test-slave-1555"
20:00:46.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:46.292941 15317 fs_report.cc:389] FS layout report
20:00:46.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:00:46.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-0/wal
20:00:46.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-0/wal
20:00:46.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-0/data/data
20:00:46.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:00:46.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:00:46.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:00:46.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:00:46.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:00:46.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:00:46.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:00:46.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:00:46.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:00:46.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:00:46.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:00:46.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:46.354818 15317 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:00:46.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:46.356606 15317 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:00:46.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:46.357131 15317 kserver.cc:163] Server-wide thread pool size limit: 3276
20:00:46.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:46.432935 15317 rpc_server.cc:307] RPC server started. Bound to: 127.14.242.126:44505
20:00:46.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:46.433005 15384 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.14.242.126:44505 every 8 connection(s)
20:00:46.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:46.435708 15317 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-0/data/info.pb
20:00:46.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:46.441738 15385 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:00:46.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:46.443446 15305 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_muOGM/build/tsan/bin/kudu as pid 15317
20:00:46.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:46.443899 15305 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-0/wal/instance
20:00:46.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:46.447273 15305 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_muOGM/build/tsan/bin/kudu
20:00:46.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_muOGM/build/tsan/bin/kudu
20:00:46.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-1/wal
20:00:46.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-1/data
20:00:46.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:00:46.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:00:46.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:00:46.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:00:46.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:00:46.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:00:46.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-1/logs
20:00:46.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-1/data/info.pb
20:00:46.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:00:46.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:00:46.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:00:46.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:00:46.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:00:46.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:00:46.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:00:46.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:00:46.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:00:46.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:00:46.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:00:46.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:00:46.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.242.125:42633
20:00:46.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.242.125
20:00:46.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:00:46.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.242.84:46193
20:00:46.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:00:46.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:00:46.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:00:46.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:00:46.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.242.126:44505,127.14.242.125:42633,127.14.242.124:32797
20:00:46.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:00:46.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35507/jwks.json with env {}
20:00:46.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:46.455837 15385 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.242.126" port: 44505 } has no permanent_uuid. Determining permanent_uuid...
20:00:46.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:46.488658 15385 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.242.125" port: 42633 } has no permanent_uuid. Determining permanent_uuid...
20:00:46.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:46.492175 15335 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.14.242.125:42633: connect: Connection refused (error 111)
20:00:46.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:46.497447 15385 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.14.242.125:42633: Network error: Client connection negotiation failed: client connection to 127.14.242.125:42633: connect: Connection refused (error 111)
20:00:46.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:46.547101 15385 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.242.125" port: 42633 } attempt: 1
20:00:46.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:46.552170 15385 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.14.242.125:42633: Network error: Client connection negotiation failed: client connection to 127.14.242.125:42633: connect: Connection refused (error 111)
20:00:46.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:46.620636 15385 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.242.125" port: 42633 } attempt: 2
20:00:46.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:46.625561 15385 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.14.242.125:42633: Network error: Client connection negotiation failed: client connection to 127.14.242.125:42633: connect: Connection refused (error 111)
20:00:46.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:46.716995 15385 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.242.125" port: 42633 } attempt: 3
20:00:46.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:46.721262 15385 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.14.242.125:42633: Network error: Client connection negotiation failed: client connection to 127.14.242.125:42633: connect: Connection refused (error 111)
20:00:46.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:46.784941 15387 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:00:46.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:46.785499 15387 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:00:46.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:46.786028 15387 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:00:46.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:46.820156 15387 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:00:46.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:46.820451 15387 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:00:46.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:46.820645 15387 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:00:46.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:46.820858 15387 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:00:46.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:46.821045 15387 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:00:46.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:46.821206 15387 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:35507/jwks.json
20:00:46.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:46.864704 15385 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.242.125" port: 42633 } attempt: 4
20:00:46.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:46.869729 15385 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.14.242.125:42633: Network error: Client connection negotiation failed: client connection to 127.14.242.125:42633: connect: Connection refused (error 111)
20:00:46.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:46.869971 15387 master_runner.cc:387] Master server non-default flags:
20:00:46.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-1/logs
20:00:46.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:00:46.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:00:46.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:00:46.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.242.84:46193
20:00:46.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:00:46.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:00:46.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-1/data
20:00:46.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-1/wal
20:00:46.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:00:46.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.242.126:44505,127.14.242.125:42633,127.14.242.124:32797
20:00:46.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:00:46.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:00:46.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:00:46.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.242.125:42633
20:00:46.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:00:46.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:00:46.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:00:46.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35507/jwks.json
20:00:46.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:00:46.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:00:46.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-1/data/info.pb
20:00:46.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.242.125
20:00:46.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:00:46.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:00:46.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:00:46.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:00:46.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:00:46.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:00:46.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:00:46.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:00:46.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:00:46.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3b18f0139b1d21a072b7353269301fba16dca54e
20:00:46.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:00:46.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Sep 2026 19:43:26 UTC on e7f111948823
20:00:46.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14030
20:00:46.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:00:46.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:46.871723 15387 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:00:46.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:46.874050 15387 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:00:46.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:46.887313 15396 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:47.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:47.169196 15385 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.242.125" port: 42633 } attempt: 5
20:00:47.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:47.175118 15385 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.14.242.125:42633: Network error: Client connection negotiation failed: client connection to 127.14.242.125:42633: connect: Connection refused (error 111)
20:00:47.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:47.722610 15385 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.242.125" port: 42633 } attempt: 6
20:00:47.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:47.730515 15385 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.14.242.125:42633: Network error: Client connection negotiation failed: client connection to 127.14.242.125:42633: connect: Connection refused (error 111)
20:00:48.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:46.889348 15397 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:48.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:46.898257 15399 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:48.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:48.098997 15398 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
20:00:48.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:48.099115 15387 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:00:48.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:48.104146 15387 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:00:48.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:48.107306 15387 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:00:48.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:48.108740 15387 hybrid_clock.cc:648] HybridClock initialized: now 1790539248108704 us; error 43 us; skew 500 ppm
20:00:48.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:48.109845 15387 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:00:48.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:48.116652 15387 webserver.cc:533] Webserver started at http://127.14.242.125:37275/ using document root <none> and password file <none>
20:00:48.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:48.117728 15387 fs_manager.cc:362] Metadata directory not provided
20:00:48.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:48.117991 15387 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:00:48.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:48.118705 15387 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:00:48.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:48.123593 15387 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-1/data/instance:
20:00:48.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6991f715c7ad4e6497b1bc10757326fc"
20:00:48.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:00:48 on dist-test-slave-1555"
20:00:48.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:48.125118 15387 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-1/wal/instance:
20:00:48.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6991f715c7ad4e6497b1bc10757326fc"
20:00:48.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:00:48 on dist-test-slave-1555"
20:00:48.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:48.133285 15387 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.008s sys 0.000s
20:00:48.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:48.139240 15408 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:00:48.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:48.140662 15387 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
20:00:48.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:48.141098 15387 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-1/data,/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-1/wal
20:00:48.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6991f715c7ad4e6497b1bc10757326fc"
20:00:48.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:00:48 on dist-test-slave-1555"
20:00:48.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:48.141574 15387 fs_report.cc:389] FS layout report
20:00:48.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:00:48.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-1/wal
20:00:48.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-1/wal
20:00:48.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-1/data/data
20:00:48.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:00:48.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:00:48.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:00:48.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:00:48.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:00:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:00:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:00:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:00:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:00:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:00:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:00:48.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:48.203545 15387 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:00:48.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:48.205304 15387 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:00:48.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:48.205840 15387 kserver.cc:163] Server-wide thread pool size limit: 3276
20:00:48.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:48.277271 15387 rpc_server.cc:307] RPC server started. Bound to: 127.14.242.125:42633
20:00:48.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:48.277388 15459 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.14.242.125:42633 every 8 connection(s)
20:00:48.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:48.280278 15387 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-1/data/info.pb
20:00:48.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:48.287420 15460 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:00:48.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:48.287694 15305 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_muOGM/build/tsan/bin/kudu as pid 15387
20:00:48.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:48.288169 15305 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-1/wal/instance
20:00:48.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:48.291157 15305 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_muOGM/build/tsan/bin/kudu
20:00:48.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_muOGM/build/tsan/bin/kudu
20:00:48.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-2/wal
20:00:48.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-2/data
20:00:48.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:00:48.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:00:48.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:00:48.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:00:48.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:00:48.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:00:48.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-2/logs
20:00:48.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-2/data/info.pb
20:00:48.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:00:48.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:00:48.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:00:48.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:00:48.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:00:48.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:00:48.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:00:48.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:00:48.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:00:48.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:00:48.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:00:48.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:00:48.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.242.124:32797
20:00:48.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.242.124
20:00:48.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:00:48.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.242.84:46193
20:00:48.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:00:48.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:00:48.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:00:48.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:00:48.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.242.126:44505,127.14.242.125:42633,127.14.242.124:32797
20:00:48.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:00:48.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35507/jwks.json with env {}
20:00:48.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:48.302824 15460 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.242.126" port: 44505 } has no permanent_uuid. Determining permanent_uuid...
20:00:48.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:48.328228 15460 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.242.125" port: 42633 } has no permanent_uuid. Determining permanent_uuid...
20:00:48.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:48.342772 15460 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.242.124" port: 32797 } has no permanent_uuid. Determining permanent_uuid...
20:00:48.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:48.345649 15411 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.14.242.124:32797: connect: Connection refused (error 111)
20:00:48.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:48.350055 15460 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.14.242.124:32797: Network error: Client connection negotiation failed: client connection to 127.14.242.124:32797: connect: Connection refused (error 111)
20:00:48.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:48.399685 15460 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.242.124" port: 32797 } attempt: 1
20:00:48.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:48.404155 15460 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.14.242.124:32797: Network error: Client connection negotiation failed: client connection to 127.14.242.124:32797: connect: Connection refused (error 111)
20:00:48.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:48.472569 15460 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.242.124" port: 32797 } attempt: 2
20:00:48.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:48.477053 15460 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.14.242.124:32797: Network error: Client connection negotiation failed: client connection to 127.14.242.124:32797: connect: Connection refused (error 111)
20:00:48.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:48.568440 15460 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.242.124" port: 32797 } attempt: 3
20:00:48.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:48.573004 15460 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.14.242.124:32797: Network error: Client connection negotiation failed: client connection to 127.14.242.124:32797: connect: Connection refused (error 111)
20:00:48.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:48.617434 15462 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:00:48.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:48.618001 15462 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:00:48.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:48.618438 15462 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:00:48.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:48.653630 15462 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:00:48.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:48.653944 15462 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:00:48.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:48.654142 15462 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:00:48.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:48.654387 15462 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:00:48.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:48.654595 15462 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:00:48.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:48.654775 15462 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:35507/jwks.json
20:00:48.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:48.691745 15462 master_runner.cc:387] Master server non-default flags:
20:00:48.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-2/logs
20:00:48.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:00:48.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:00:48.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:00:48.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.242.84:46193
20:00:48.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:00:48.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:00:48.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-2/data
20:00:48.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-2/wal
20:00:48.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:00:48.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.242.126:44505,127.14.242.125:42633,127.14.242.124:32797
20:00:48.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:00:48.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:00:48.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:00:48.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.242.124:32797
20:00:48.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:00:48.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:00:48.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:00:48.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35507/jwks.json
20:00:48.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:00:48.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:00:48.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-2/data/info.pb
20:00:48.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.242.124
20:00:48.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:00:48.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:00:48.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:00:48.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:00:48.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:00:48.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:00:48.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:00:48.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:00:48.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:00:48.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3b18f0139b1d21a072b7353269301fba16dca54e
20:00:48.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:00:48.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Sep 2026 19:43:26 UTC on e7f111948823
20:00:48.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14030
20:00:48.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:00:48.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:48.693192 15462 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:00:48.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:48.695334 15462 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:00:48.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:48.711525 15472 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:48.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:48.716400 15460 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.242.124" port: 32797 } attempt: 4
20:00:48.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:48.722405 15460 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.14.242.124:32797: Network error: Client connection negotiation failed: client connection to 127.14.242.124:32797: connect: Connection refused (error 111)
20:00:48.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:48.791213 15385 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.242.125" port: 42633 } attempt: 7
20:00:48.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:48.809289 15385 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.242.124" port: 32797 } has no permanent_uuid. Determining permanent_uuid...
20:00:48.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:48.817848 15385 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.14.242.124:32797: Network error: Client connection negotiation failed: client connection to 127.14.242.124:32797: connect: Connection refused (error 111)
20:00:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:48.876231 15385 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.242.124" port: 32797 } attempt: 1
20:00:48.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:48.881049 15385 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.14.242.124:32797: Network error: Client connection negotiation failed: client connection to 127.14.242.124:32797: connect: Connection refused (error 111)
20:00:48.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:48.962673 15385 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.242.124" port: 32797 } attempt: 2
20:00:48.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:48.967386 15385 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.14.242.124:32797: Network error: Client connection negotiation failed: client connection to 127.14.242.124:32797: connect: Connection refused (error 111)
20:00:49.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:49.021875 15460 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.242.124" port: 32797 } attempt: 5
20:00:49.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:49.026247 15460 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.14.242.124:32797: Network error: Client connection negotiation failed: client connection to 127.14.242.124:32797: connect: Connection refused (error 111)
20:00:49.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:49.052990 15385 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.242.124" port: 32797 } attempt: 3
20:00:49.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:49.058382 15385 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.14.242.124:32797: Network error: Client connection negotiation failed: client connection to 127.14.242.124:32797: connect: Connection refused (error 111)
20:00:49.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:49.198807 15385 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.242.124" port: 32797 } attempt: 4
20:00:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:49.204000 15385 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.14.242.124:32797: Network error: Client connection negotiation failed: client connection to 127.14.242.124:32797: connect: Connection refused (error 111)
20:00:49.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:49.487535 15385 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.242.124" port: 32797 } attempt: 5
20:00:49.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:49.492828 15385 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.14.242.124:32797: Network error: Client connection negotiation failed: client connection to 127.14.242.124:32797: connect: Connection refused (error 111)
20:00:49.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:49.573678 15460 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.242.124" port: 32797 } attempt: 6
20:00:49.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:49.582199 15460 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.14.242.124:32797: Network error: Client connection negotiation failed: client connection to 127.14.242.124:32797: connect: Connection refused (error 111)
20:00:49.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:48.715255 15475 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:49.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:48.717789 15462 server_base.cc:1061] running on GCE node
20:00:49.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:48.715660 15473 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:49.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:49.936178 15462 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:00:49.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:49.939428 15462 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:00:49.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:49.940928 15462 hybrid_clock.cc:648] HybridClock initialized: now 1790539249940860 us; error 79 us; skew 500 ppm
20:00:49.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:49.941969 15462 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:00:49.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:49.948904 15462 webserver.cc:533] Webserver started at http://127.14.242.124:36617/ using document root <none> and password file <none>
20:00:49.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:49.950114 15462 fs_manager.cc:362] Metadata directory not provided
20:00:49.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:49.950342 15462 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:00:49.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:49.950868 15462 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:00:49.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:49.955513 15462 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-2/data/instance:
20:00:49.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5d0e120d3e514e47ac24ffd17df3b316"
20:00:49.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:00:49 on dist-test-slave-1555"
20:00:49.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:49.956784 15462 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-2/wal/instance:
20:00:49.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5d0e120d3e514e47ac24ffd17df3b316"
20:00:49.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:00:49 on dist-test-slave-1555"
20:00:49.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:49.964641 15462 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
20:00:49.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:49.970772 15486 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:00:49.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:49.972127 15462 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.003s
20:00:49.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:49.972421 15462 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-2/data,/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-2/wal
20:00:49.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5d0e120d3e514e47ac24ffd17df3b316"
20:00:49.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:00:49 on dist-test-slave-1555"
20:00:49.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:49.972815 15462 fs_report.cc:389] FS layout report
20:00:49.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:00:49.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-2/wal
20:00:49.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-2/wal
20:00:49.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-2/data/data
20:00:49.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:00:49.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:00:49.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:00:49.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:00:49.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:00:49.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:00:49.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:00:49.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:00:49.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:00:49.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:00:49.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:00:50.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.032413 15462 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:00:50.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.034101 15462 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:00:50.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.034631 15462 kserver.cc:163] Server-wide thread pool size limit: 3276
20:00:50.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.045285 15385 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.242.124" port: 32797 } attempt: 6
20:00:50.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:50.053650 15385 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.14.242.124:32797: Network error: Client connection negotiation failed: client connection to 127.14.242.124:32797: connect: Connection refused (error 111)
20:00:50.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.113686 15462 rpc_server.cc:307] RPC server started. Bound to: 127.14.242.124:32797
20:00:50.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.113783 15539 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.14.242.124:32797 every 8 connection(s)
20:00:50.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.116382 15462 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-2/data/info.pb
20:00:50.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.118003 15305 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_muOGM/build/tsan/bin/kudu as pid 15462
20:00:50.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.118552 15305 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/master-2/wal/instance
20:00:50.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.124033 15540 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:00:50.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.141203 15540 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.242.126" port: 44505 } has no permanent_uuid. Determining permanent_uuid...
20:00:50.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.166986 15540 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.242.125" port: 42633 } has no permanent_uuid. Determining permanent_uuid...
20:00:50.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.178593 15540 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.242.124" port: 32797 } has no permanent_uuid. Determining permanent_uuid...
20:00:50.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.196547 15540 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5d0e120d3e514e47ac24ffd17df3b316: Bootstrap starting.
20:00:50.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.203186 15540 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 5d0e120d3e514e47ac24ffd17df3b316: Neither blocks nor log segments found. Creating new log.
20:00:50.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.207154 15540 log.cc:826] T 00000000000000000000000000000000 P 5d0e120d3e514e47ac24ffd17df3b316: Log is configured to *not* fsync() on all Append() calls
20:00:50.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.211761 15540 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5d0e120d3e514e47ac24ffd17df3b316: No bootstrap required, opened a new log
20:00:50.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.231666 15540 raft_consensus.cc:359] T 00000000000000000000000000000000 P 5d0e120d3e514e47ac24ffd17df3b316 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b00bb10f6da46b8ab5a69d8904a4fc5" member_type: VOTER last_known_addr { host: "127.14.242.126" port: 44505 } } peers { permanent_uuid: "6991f715c7ad4e6497b1bc10757326fc" member_type: VOTER last_known_addr { host: "127.14.242.125" port: 42633 } } peers { permanent_uuid: "5d0e120d3e514e47ac24ffd17df3b316" member_type: VOTER last_known_addr { host: "127.14.242.124" port: 32797 } }
20:00:50.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.232350 15540 raft_consensus.cc:385] T 00000000000000000000000000000000 P 5d0e120d3e514e47ac24ffd17df3b316 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:00:50.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.232616 15540 raft_consensus.cc:740] T 00000000000000000000000000000000 P 5d0e120d3e514e47ac24ffd17df3b316 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5d0e120d3e514e47ac24ffd17df3b316, State: Initialized, Role: FOLLOWER
20:00:50.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.233497 15540 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5d0e120d3e514e47ac24ffd17df3b316 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b00bb10f6da46b8ab5a69d8904a4fc5" member_type: VOTER last_known_addr { host: "127.14.242.126" port: 44505 } } peers { permanent_uuid: "6991f715c7ad4e6497b1bc10757326fc" member_type: VOTER last_known_addr { host: "127.14.242.125" port: 42633 } } peers { permanent_uuid: "5d0e120d3e514e47ac24ffd17df3b316" member_type: VOTER last_known_addr { host: "127.14.242.124" port: 32797 } }
20:00:50.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.235934 15549 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5d0e120d3e514e47ac24ffd17df3b316 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b00bb10f6da46b8ab5a69d8904a4fc5" member_type: VOTER last_known_addr { host: "127.14.242.126" port: 44505 } } peers { permanent_uuid: "6991f715c7ad4e6497b1bc10757326fc" member_type: VOTER last_known_addr { host: "127.14.242.125" port: 42633 } } peers { permanent_uuid: "5d0e120d3e514e47ac24ffd17df3b316" member_type: VOTER last_known_addr { host: "127.14.242.124" port: 32797 } } }
20:00:50.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.236593 15549 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5d0e120d3e514e47ac24ffd17df3b316 [sys.catalog]: This master's current role is: FOLLOWER
20:00:50.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.237720 15540 sys_catalog.cc:565] T 00000000000000000000000000000000 P 5d0e120d3e514e47ac24ffd17df3b316 [sys.catalog]: configured and running, proceeding with master startup.
20:00:50.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:50.265910 15560 catalog_manager.cc:1607] T 00000000000000000000000000000000 P 5d0e120d3e514e47ac24ffd17df3b316: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:00:50.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:50.266206 15560 catalog_manager.cc:896] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:00:50.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:50.290009 15492 tablet.cc:2433] T 00000000000000000000000000000000 P 5d0e120d3e514e47ac24ffd17df3b316: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:00:50.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.262837 15462 master_runner.cc:187] Error getting master registration for 127.14.242.126:44505: OK, instance_id { permanent_uuid: "6b00bb10f6da46b8ab5a69d8904a4fc5" instance_seqno: 1790539246412633 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:00:50.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.294621 15462 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:00:50.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.414095 15549 raft_consensus.cc:493] T 00000000000000000000000000000000 P 5d0e120d3e514e47ac24ffd17df3b316 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:00:50.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.414673 15549 raft_consensus.cc:515] T 00000000000000000000000000000000 P 5d0e120d3e514e47ac24ffd17df3b316 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b00bb10f6da46b8ab5a69d8904a4fc5" member_type: VOTER last_known_addr { host: "127.14.242.126" port: 44505 } } peers { permanent_uuid: "6991f715c7ad4e6497b1bc10757326fc" member_type: VOTER last_known_addr { host: "127.14.242.125" port: 42633 } } peers { permanent_uuid: "5d0e120d3e514e47ac24ffd17df3b316" member_type: VOTER last_known_addr { host: "127.14.242.124" port: 32797 } }
20:00:50.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.417431 15549 leader_election.cc:290] T 00000000000000000000000000000000 P 5d0e120d3e514e47ac24ffd17df3b316 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6b00bb10f6da46b8ab5a69d8904a4fc5 (127.14.242.126:44505), 6991f715c7ad4e6497b1bc10757326fc (127.14.242.125:42633)
20:00:50.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.642623 15460 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.242.124" port: 32797 } attempt: 7
20:00:50.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.679596 15460 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6991f715c7ad4e6497b1bc10757326fc: Bootstrap starting.
20:00:50.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.687879 15460 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 6991f715c7ad4e6497b1bc10757326fc: Neither blocks nor log segments found. Creating new log.
20:00:50.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.690189 15460 log.cc:826] T 00000000000000000000000000000000 P 6991f715c7ad4e6497b1bc10757326fc: Log is configured to *not* fsync() on all Append() calls
20:00:50.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.695499 15460 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6991f715c7ad4e6497b1bc10757326fc: No bootstrap required, opened a new log
20:00:50.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.727659 15460 raft_consensus.cc:359] T 00000000000000000000000000000000 P 6991f715c7ad4e6497b1bc10757326fc [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b00bb10f6da46b8ab5a69d8904a4fc5" member_type: VOTER last_known_addr { host: "127.14.242.126" port: 44505 } } peers { permanent_uuid: "6991f715c7ad4e6497b1bc10757326fc" member_type: VOTER last_known_addr { host: "127.14.242.125" port: 42633 } } peers { permanent_uuid: "5d0e120d3e514e47ac24ffd17df3b316" member_type: VOTER last_known_addr { host: "127.14.242.124" port: 32797 } }
20:00:50.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.728561 15460 raft_consensus.cc:385] T 00000000000000000000000000000000 P 6991f715c7ad4e6497b1bc10757326fc [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:00:50.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.728855 15460 raft_consensus.cc:740] T 00000000000000000000000000000000 P 6991f715c7ad4e6497b1bc10757326fc [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6991f715c7ad4e6497b1bc10757326fc, State: Initialized, Role: FOLLOWER
20:00:50.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.730189 15460 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6991f715c7ad4e6497b1bc10757326fc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b00bb10f6da46b8ab5a69d8904a4fc5" member_type: VOTER last_known_addr { host: "127.14.242.126" port: 44505 } } peers { permanent_uuid: "6991f715c7ad4e6497b1bc10757326fc" member_type: VOTER last_known_addr { host: "127.14.242.125" port: 42633 } } peers { permanent_uuid: "5d0e120d3e514e47ac24ffd17df3b316" member_type: VOTER last_known_addr { host: "127.14.242.124" port: 32797 } }
20:00:50.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.735984 15564 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6991f715c7ad4e6497b1bc10757326fc [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b00bb10f6da46b8ab5a69d8904a4fc5" member_type: VOTER last_known_addr { host: "127.14.242.126" port: 44505 } } peers { permanent_uuid: "6991f715c7ad4e6497b1bc10757326fc" member_type: VOTER last_known_addr { host: "127.14.242.125" port: 42633 } } peers { permanent_uuid: "5d0e120d3e514e47ac24ffd17df3b316" member_type: VOTER last_known_addr { host: "127.14.242.124" port: 32797 } } }
20:00:50.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.737084 15460 sys_catalog.cc:565] T 00000000000000000000000000000000 P 6991f715c7ad4e6497b1bc10757326fc [sys.catalog]: configured and running, proceeding with master startup.
20:00:50.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.737288 15564 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6991f715c7ad4e6497b1bc10757326fc [sys.catalog]: This master's current role is: FOLLOWER
20:00:50.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.737257 15435 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5d0e120d3e514e47ac24ffd17df3b316" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6991f715c7ad4e6497b1bc10757326fc" is_pre_election: true
20:00:50.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.738946 15435 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 6991f715c7ad4e6497b1bc10757326fc [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5d0e120d3e514e47ac24ffd17df3b316 in term 0.
20:00:50.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.740628 15488 leader_election.cc:304] T 00000000000000000000000000000000 P 5d0e120d3e514e47ac24ffd17df3b316 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5d0e120d3e514e47ac24ffd17df3b316, 6991f715c7ad4e6497b1bc10757326fc; no voters:
20:00:50.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.741930 15549 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 5d0e120d3e514e47ac24ffd17df3b316 [term 0 FOLLOWER]: Leader pre-election won for term 1
20:00:50.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.742389 15549 raft_consensus.cc:493] T 00000000000000000000000000000000 P 5d0e120d3e514e47ac24ffd17df3b316 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:00:50.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.742720 15549 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 5d0e120d3e514e47ac24ffd17df3b316 [term 0 FOLLOWER]: Advancing to term 1
20:00:50.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.750635 15549 raft_consensus.cc:515] T 00000000000000000000000000000000 P 5d0e120d3e514e47ac24ffd17df3b316 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b00bb10f6da46b8ab5a69d8904a4fc5" member_type: VOTER last_known_addr { host: "127.14.242.126" port: 44505 } } peers { permanent_uuid: "6991f715c7ad4e6497b1bc10757326fc" member_type: VOTER last_known_addr { host: "127.14.242.125" port: 42633 } } peers { permanent_uuid: "5d0e120d3e514e47ac24ffd17df3b316" member_type: VOTER last_known_addr { host: "127.14.242.124" port: 32797 } }
20:00:50.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.753516 15549 leader_election.cc:290] T 00000000000000000000000000000000 P 5d0e120d3e514e47ac24ffd17df3b316 [CANDIDATE]: Term 1 election: Requested vote from peers 6b00bb10f6da46b8ab5a69d8904a4fc5 (127.14.242.126:44505), 6991f715c7ad4e6497b1bc10757326fc (127.14.242.125:42633)
20:00:50.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.754630 15435 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5d0e120d3e514e47ac24ffd17df3b316" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6991f715c7ad4e6497b1bc10757326fc"
20:00:50.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.755206 15435 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 6991f715c7ad4e6497b1bc10757326fc [term 0 FOLLOWER]: Advancing to term 1
20:00:50.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.762980 15435 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 6991f715c7ad4e6497b1bc10757326fc [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5d0e120d3e514e47ac24ffd17df3b316 in term 1.
20:00:50.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.764344 15488 leader_election.cc:304] T 00000000000000000000000000000000 P 5d0e120d3e514e47ac24ffd17df3b316 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5d0e120d3e514e47ac24ffd17df3b316, 6991f715c7ad4e6497b1bc10757326fc; no voters:
20:00:50.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.765215 15549 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 5d0e120d3e514e47ac24ffd17df3b316 [term 1 FOLLOWER]: Leader election won for term 1
20:00:50.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.771149 15549 raft_consensus.cc:697] T 00000000000000000000000000000000 P 5d0e120d3e514e47ac24ffd17df3b316 [term 1 LEADER]: Becoming Leader. State: Replica: 5d0e120d3e514e47ac24ffd17df3b316, State: Running, Role: LEADER
20:00:50.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.773003 15549 consensus_queue.cc:237] T 00000000000000000000000000000000 P 5d0e120d3e514e47ac24ffd17df3b316 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b00bb10f6da46b8ab5a69d8904a4fc5" member_type: VOTER last_known_addr { host: "127.14.242.126" port: 44505 } } peers { permanent_uuid: "6991f715c7ad4e6497b1bc10757326fc" member_type: VOTER last_known_addr { host: "127.14.242.125" port: 42633 } } peers { permanent_uuid: "5d0e120d3e514e47ac24ffd17df3b316" member_type: VOTER last_known_addr { host: "127.14.242.124" port: 32797 } }
20:00:50.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.790793 15571 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5d0e120d3e514e47ac24ffd17df3b316 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 5d0e120d3e514e47ac24ffd17df3b316. Latest consensus state: current_term: 1 leader_uuid: "5d0e120d3e514e47ac24ffd17df3b316" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b00bb10f6da46b8ab5a69d8904a4fc5" member_type: VOTER last_known_addr { host: "127.14.242.126" port: 44505 } } peers { permanent_uuid: "6991f715c7ad4e6497b1bc10757326fc" member_type: VOTER last_known_addr { host: "127.14.242.125" port: 42633 } } peers { permanent_uuid: "5d0e120d3e514e47ac24ffd17df3b316" member_type: VOTER last_known_addr { host: "127.14.242.124" port: 32797 } } }
20:00:50.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.792784 15571 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5d0e120d3e514e47ac24ffd17df3b316 [sys.catalog]: This master's current role is: LEADER
20:00:50.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.799531 15576 catalog_manager.cc:1524] Loading table and tablet metadata into memory...
20:00:50.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.807694 15578 catalog_manager.cc:818] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
20:00:50.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.807780 15576 catalog_manager.cc:1533] Initializing Kudu cluster ID...
20:00:50.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.835263 15435 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 6991f715c7ad4e6497b1bc10757326fc [term 1 FOLLOWER]: Refusing update from remote peer 5d0e120d3e514e47ac24ffd17df3b316: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:00:50.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.837158 15571 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 5d0e120d3e514e47ac24ffd17df3b316 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6991f715c7ad4e6497b1bc10757326fc" member_type: VOTER last_known_addr { host: "127.14.242.125" port: 42633 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:00:50.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.812462 15387 master_runner.cc:187] Error getting master registration for 127.14.242.126:44505: OK, instance_id { permanent_uuid: "6b00bb10f6da46b8ab5a69d8904a4fc5" instance_seqno: 1790539246412633 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:00:50.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.854070 15387 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:00:50.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.866420 15579 mvcc.cc:204] Tried to move back new op lower bound from 7334048771383406592 to 7334048771182223360. Current Snapshot: MvccSnapshot[applied={T|T < 7334048771383406592}]
20:00:50.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.865119 15564 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6991f715c7ad4e6497b1bc10757326fc [sys.catalog]: SysCatalogTable state changed. Reason: New leader 5d0e120d3e514e47ac24ffd17df3b316. Latest consensus state: current_term: 1 leader_uuid: "5d0e120d3e514e47ac24ffd17df3b316" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b00bb10f6da46b8ab5a69d8904a4fc5" member_type: VOTER last_known_addr { host: "127.14.242.126" port: 44505 } } peers { permanent_uuid: "6991f715c7ad4e6497b1bc10757326fc" member_type: VOTER last_known_addr { host: "127.14.242.125" port: 42633 } } peers { permanent_uuid: "5d0e120d3e514e47ac24ffd17df3b316" member_type: VOTER last_known_addr { host: "127.14.242.124" port: 32797 } } }
20:00:50.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.867183 15564 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6991f715c7ad4e6497b1bc10757326fc [sys.catalog]: This master's current role is: FOLLOWER
20:00:50.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.871176 15576 catalog_manager.cc:1396] Generated new cluster ID: f58197f2355943edafbcc6ae188fe03d
20:00:50.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.871381 15576 catalog_manager.cc:1544] Initializing Kudu internal certificate authority...
20:00:50.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.873425 15549 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5d0e120d3e514e47ac24ffd17df3b316 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "5d0e120d3e514e47ac24ffd17df3b316" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b00bb10f6da46b8ab5a69d8904a4fc5" member_type: VOTER last_known_addr { host: "127.14.242.126" port: 44505 } } peers { permanent_uuid: "6991f715c7ad4e6497b1bc10757326fc" member_type: VOTER last_known_addr { host: "127.14.242.125" port: 42633 } } peers { permanent_uuid: "5d0e120d3e514e47ac24ffd17df3b316" member_type: VOTER last_known_addr { host: "127.14.242.124" port: 32797 } } }
20:00:50.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.874541 15549 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5d0e120d3e514e47ac24ffd17df3b316 [sys.catalog]: This master's current role is: LEADER
20:00:50.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.894701 15564 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6991f715c7ad4e6497b1bc10757326fc [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "5d0e120d3e514e47ac24ffd17df3b316" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b00bb10f6da46b8ab5a69d8904a4fc5" member_type: VOTER last_known_addr { host: "127.14.242.126" port: 44505 } } peers { permanent_uuid: "6991f715c7ad4e6497b1bc10757326fc" member_type: VOTER last_known_addr { host: "127.14.242.125" port: 42633 } } peers { permanent_uuid: "5d0e120d3e514e47ac24ffd17df3b316" member_type: VOTER last_known_addr { host: "127.14.242.124" port: 32797 } } }
20:00:50.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.896855 15564 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6991f715c7ad4e6497b1bc10757326fc [sys.catalog]: This master's current role is: FOLLOWER
20:00:50.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.910400 15576 catalog_manager.cc:1419] Generated new certificate authority record
20:00:50.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.912501 15576 catalog_manager.cc:1553] Loading token signing keys...
20:00:50.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.937489 15576 catalog_manager.cc:6219] T 00000000000000000000000000000000 P 5d0e120d3e514e47ac24ffd17df3b316: Generated new TSK 0
20:00:50.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.938602 15576 catalog_manager.cc:1563] Initializing in-progress tserver states...
20:00:50.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:50.964365 15305 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_muOGM/build/tsan/bin/kudu
20:00:50.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_muOGM/build/tsan/bin/kudu
20:00:50.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-0/wal
20:00:50.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-0/data
20:00:50.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:00:50.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:00:50.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:00:50.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:00:50.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:00:50.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:00:50.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-0/logs
20:00:50.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-0/data/info.pb
20:00:50.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:00:50.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:00:50.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:00:50.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:00:50.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:00:50.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:00:50.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:00:50.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:00:50.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:00:50.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:00:50.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.242.65:0
20:00:50.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.14.242.65
20:00:50.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.242.65
20:00:50.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:00:50.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.14.242.126:44505,127.14.242.125:42633,127.14.242.124:32797
20:00:50.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.242.84:46193
20:00:50.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:00:50.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:00:50.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:00:51.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:51.087133 15385 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.242.124" port: 32797 } attempt: 7
20:00:51.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:51.116730 15385 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6b00bb10f6da46b8ab5a69d8904a4fc5: Bootstrap starting.
20:00:51.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:51.125097 15385 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 6b00bb10f6da46b8ab5a69d8904a4fc5: Neither blocks nor log segments found. Creating new log.
20:00:51.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:51.127997 15385 log.cc:826] T 00000000000000000000000000000000 P 6b00bb10f6da46b8ab5a69d8904a4fc5: Log is configured to *not* fsync() on all Append() calls
20:00:51.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:51.139591 15385 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6b00bb10f6da46b8ab5a69d8904a4fc5: No bootstrap required, opened a new log
20:00:51.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:51.169704 15385 raft_consensus.cc:359] T 00000000000000000000000000000000 P 6b00bb10f6da46b8ab5a69d8904a4fc5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b00bb10f6da46b8ab5a69d8904a4fc5" member_type: VOTER last_known_addr { host: "127.14.242.126" port: 44505 } } peers { permanent_uuid: "6991f715c7ad4e6497b1bc10757326fc" member_type: VOTER last_known_addr { host: "127.14.242.125" port: 42633 } } peers { permanent_uuid: "5d0e120d3e514e47ac24ffd17df3b316" member_type: VOTER last_known_addr { host: "127.14.242.124" port: 32797 } }
20:00:51.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:51.171142 15385 raft_consensus.cc:385] T 00000000000000000000000000000000 P 6b00bb10f6da46b8ab5a69d8904a4fc5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:00:51.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:51.171717 15385 raft_consensus.cc:740] T 00000000000000000000000000000000 P 6b00bb10f6da46b8ab5a69d8904a4fc5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6b00bb10f6da46b8ab5a69d8904a4fc5, State: Initialized, Role: FOLLOWER
20:00:51.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:51.173169 15385 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6b00bb10f6da46b8ab5a69d8904a4fc5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b00bb10f6da46b8ab5a69d8904a4fc5" member_type: VOTER last_known_addr { host: "127.14.242.126" port: 44505 } } peers { permanent_uuid: "6991f715c7ad4e6497b1bc10757326fc" member_type: VOTER last_known_addr { host: "127.14.242.125" port: 42633 } } peers { permanent_uuid: "5d0e120d3e514e47ac24ffd17df3b316" member_type: VOTER last_known_addr { host: "127.14.242.124" port: 32797 } }
20:00:51.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:51.182080 15587 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6b00bb10f6da46b8ab5a69d8904a4fc5 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b00bb10f6da46b8ab5a69d8904a4fc5" member_type: VOTER last_known_addr { host: "127.14.242.126" port: 44505 } } peers { permanent_uuid: "6991f715c7ad4e6497b1bc10757326fc" member_type: VOTER last_known_addr { host: "127.14.242.125" port: 42633 } } peers { permanent_uuid: "5d0e120d3e514e47ac24ffd17df3b316" member_type: VOTER last_known_addr { host: "127.14.242.124" port: 32797 } } }
20:00:51.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:51.193740 15358 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 6b00bb10f6da46b8ab5a69d8904a4fc5 [term 0 FOLLOWER]: Advancing to term 1
20:00:51.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:51.194216 15360 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5d0e120d3e514e47ac24ffd17df3b316" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6b00bb10f6da46b8ab5a69d8904a4fc5" is_pre_election: true
20:00:51.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:51.194880 15359 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5d0e120d3e514e47ac24ffd17df3b316" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6b00bb10f6da46b8ab5a69d8904a4fc5"
20:00:51.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:51.206193 15358 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 6b00bb10f6da46b8ab5a69d8904a4fc5 [term 1 FOLLOWER]: Refusing update from remote peer 5d0e120d3e514e47ac24ffd17df3b316: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:00:51.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:51.193104 15385 sys_catalog.cc:565] T 00000000000000000000000000000000 P 6b00bb10f6da46b8ab5a69d8904a4fc5 [sys.catalog]: configured and running, proceeding with master startup.
20:00:51.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:51.208448 15571 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 5d0e120d3e514e47ac24ffd17df3b316 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b00bb10f6da46b8ab5a69d8904a4fc5" member_type: VOTER last_known_addr { host: "127.14.242.126" port: 44505 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:00:51.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:51.215333 15587 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6b00bb10f6da46b8ab5a69d8904a4fc5 [sys.catalog]: This master's current role is: FOLLOWER
20:00:51.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:51.272454 15590 raft_consensus.cc:493] T 00000000000000000000000000000000 P 6b00bb10f6da46b8ab5a69d8904a4fc5 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 5d0e120d3e514e47ac24ffd17df3b316)
20:00:51.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:51.273099 15590 raft_consensus.cc:515] T 00000000000000000000000000000000 P 6b00bb10f6da46b8ab5a69d8904a4fc5 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b00bb10f6da46b8ab5a69d8904a4fc5" member_type: VOTER last_known_addr { host: "127.14.242.126" port: 44505 } } peers { permanent_uuid: "6991f715c7ad4e6497b1bc10757326fc" member_type: VOTER last_known_addr { host: "127.14.242.125" port: 42633 } } peers { permanent_uuid: "5d0e120d3e514e47ac24ffd17df3b316" member_type: VOTER last_known_addr { host: "127.14.242.124" port: 32797 } }
20:00:51.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:51.275770 15588 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6b00bb10f6da46b8ab5a69d8904a4fc5 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "5d0e120d3e514e47ac24ffd17df3b316" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b00bb10f6da46b8ab5a69d8904a4fc5" member_type: VOTER last_known_addr { host: "127.14.242.126" port: 44505 } } peers { permanent_uuid: "6991f715c7ad4e6497b1bc10757326fc" member_type: VOTER last_known_addr { host: "127.14.242.125" port: 42633 } } peers { permanent_uuid: "5d0e120d3e514e47ac24ffd17df3b316" member_type: VOTER last_known_addr { host: "127.14.242.124" port: 32797 } } }
20:00:51.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:51.276599 15588 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6b00bb10f6da46b8ab5a69d8904a4fc5 [sys.catalog]: This master's current role is: FOLLOWER
20:00:51.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:51.284170 15435 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6b00bb10f6da46b8ab5a69d8904a4fc5" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "6991f715c7ad4e6497b1bc10757326fc" is_pre_election: true
20:00:51.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:51.284619 15515 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6b00bb10f6da46b8ab5a69d8904a4fc5" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "5d0e120d3e514e47ac24ffd17df3b316" is_pre_election: true
20:00:51.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:51.286938 15336 leader_election.cc:304] T 00000000000000000000000000000000 P 6b00bb10f6da46b8ab5a69d8904a4fc5 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6b00bb10f6da46b8ab5a69d8904a4fc5; no voters: 5d0e120d3e514e47ac24ffd17df3b316, 6991f715c7ad4e6497b1bc10757326fc
20:00:51.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:51.289770 15587 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6b00bb10f6da46b8ab5a69d8904a4fc5 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 5d0e120d3e514e47ac24ffd17df3b316. Latest consensus state: current_term: 1 leader_uuid: "5d0e120d3e514e47ac24ffd17df3b316" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b00bb10f6da46b8ab5a69d8904a4fc5" member_type: VOTER last_known_addr { host: "127.14.242.126" port: 44505 } } peers { permanent_uuid: "6991f715c7ad4e6497b1bc10757326fc" member_type: VOTER last_known_addr { host: "127.14.242.125" port: 42633 } } peers { permanent_uuid: "5d0e120d3e514e47ac24ffd17df3b316" member_type: VOTER last_known_addr { host: "127.14.242.124" port: 32797 } } }
20:00:51.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:51.290488 15587 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6b00bb10f6da46b8ab5a69d8904a4fc5 [sys.catalog]: This master's current role is: FOLLOWER
20:00:51.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:51.290926 15587 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 6b00bb10f6da46b8ab5a69d8904a4fc5 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
20:00:51.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:51.292106 15590 leader_election.cc:290] T 00000000000000000000000000000000 P 6b00bb10f6da46b8ab5a69d8904a4fc5 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 6991f715c7ad4e6497b1bc10757326fc (127.14.242.125:42633), 5d0e120d3e514e47ac24ffd17df3b316 (127.14.242.124:32797)
20:00:51.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:51.372359 15604 catalog_manager.cc:1308] Loaded cluster ID: f58197f2355943edafbcc6ae188fe03d
20:00:51.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:51.373159 15604 catalog_manager.cc:1601] T 00000000000000000000000000000000 P 6b00bb10f6da46b8ab5a69d8904a4fc5: loading cluster ID for follower catalog manager: success
20:00:51.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:51.380672 15604 catalog_manager.cc:1623] T 00000000000000000000000000000000 P 6b00bb10f6da46b8ab5a69d8904a4fc5: acquiring CA information for follower catalog manager: success
20:00:51.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:51.386737 15604 catalog_manager.cc:1651] T 00000000000000000000000000000000 P 6b00bb10f6da46b8ab5a69d8904a4fc5: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:00:51.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:51.499634 15584 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:00:51.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:51.500823 15584 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:00:51.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:51.501838 15584 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:00:51.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:51.569509 15584 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:00:51.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:51.570945 15584 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.14.242.65
20:00:51.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:51.640144 15584 tablet_server_runner.cc:78] Tablet server non-default flags:
20:00:51.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-0/logs
20:00:51.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:00:51.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:00:51.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:00:51.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.242.84:46193
20:00:51.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:00:51.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:00:51.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-0/data
20:00:51.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-0/wal
20:00:51.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:00:51.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:00:51.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.242.65:0
20:00:51.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:00:51.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:00:51.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:00:51.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-0/data/info.pb
20:00:51.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.242.65
20:00:51.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:00:51.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.14.242.126:44505,127.14.242.125:42633,127.14.242.124:32797
20:00:51.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:00:51.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:00:51.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:00:51.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:00:51.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:00:51.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.14.242.65
20:00:51.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:00:51.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:00:51.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:00:51.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3b18f0139b1d21a072b7353269301fba16dca54e
20:00:51.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:00:51.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Sep 2026 19:43:26 UTC on e7f111948823
20:00:51.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14030
20:00:51.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:00:51.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:51.641925 15584 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:00:51.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:51.644138 15584 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:00:51.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:51.660857 15611 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:51.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:51.773279 15617 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5d0e120d3e514e47ac24ffd17df3b316 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "5d0e120d3e514e47ac24ffd17df3b316" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b00bb10f6da46b8ab5a69d8904a4fc5" member_type: VOTER last_known_addr { host: "127.14.242.126" port: 44505 } } peers { permanent_uuid: "6991f715c7ad4e6497b1bc10757326fc" member_type: VOTER last_known_addr { host: "127.14.242.125" port: 42633 } } peers { permanent_uuid: "5d0e120d3e514e47ac24ffd17df3b316" member_type: VOTER last_known_addr { host: "127.14.242.124" port: 32797 } } }
20:00:51.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:51.775503 15617 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5d0e120d3e514e47ac24ffd17df3b316 [sys.catalog]: This master's current role is: LEADER
20:00:51.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:51.833235 15578 catalog_manager.cc:1308] Loaded cluster ID: f58197f2355943edafbcc6ae188fe03d
20:00:51.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:51.833973 15578 catalog_manager.cc:1601] T 00000000000000000000000000000000 P 6991f715c7ad4e6497b1bc10757326fc: loading cluster ID for follower catalog manager: success
20:00:51.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:51.841843 15578 catalog_manager.cc:1623] T 00000000000000000000000000000000 P 6991f715c7ad4e6497b1bc10757326fc: acquiring CA information for follower catalog manager: success
20:00:51.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:51.855204 15578 catalog_manager.cc:1651] T 00000000000000000000000000000000 P 6991f715c7ad4e6497b1bc10757326fc: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:00:53.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:51.673050 15615 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:53.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:53.240214 15613 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
20:00:53.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:53.240410 15584 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:00:53.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:51.670703 15612 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:53.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:53.247896 15584 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:00:53.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:53.257340 15584 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:00:53.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:53.259037 15584 hybrid_clock.cc:648] HybridClock initialized: now 1790539253258985 us; error 59 us; skew 500 ppm
20:00:53.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:53.260311 15584 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:00:53.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:53.304906 15584 webserver.cc:533] Webserver started at http://127.14.242.65:33123/ using document root <none> and password file <none>
20:00:53.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:53.306651 15584 fs_manager.cc:362] Metadata directory not provided
20:00:53.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:53.307217 15584 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:00:53.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:53.307982 15584 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:00:53.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:53.314391 15584 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-0/data/instance:
20:00:53.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a23ba18e795745cf998aaf04a40856cb"
20:00:53.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:00:53 on dist-test-slave-1555"
20:00:53.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:53.328977 15584 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-0/wal/instance:
20:00:53.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a23ba18e795745cf998aaf04a40856cb"
20:00:53.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:00:53 on dist-test-slave-1555"
20:00:53.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:53.345795 15584 fs_manager.cc:696] Time spent creating directory manager: real 0.016s user 0.012s sys 0.001s
20:00:53.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:53.359754 15625 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:00:53.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:53.361631 15584 fs_manager.cc:730] Time spent opening block manager: real 0.011s user 0.005s sys 0.004s
20:00:53.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:53.362291 15584 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-0/data,/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-0/wal
20:00:53.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a23ba18e795745cf998aaf04a40856cb"
20:00:53.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:00:53 on dist-test-slave-1555"
20:00:53.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:53.363095 15584 fs_report.cc:389] FS layout report
20:00:53.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:00:53.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-0/wal
20:00:53.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-0/wal
20:00:53.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-0/data/data
20:00:53.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:00:53.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:00:53.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:00:53.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:00:53.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:00:53.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:00:53.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:00:53.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:00:53.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:00:53.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:00:53.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:00:53.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:53.476482 15584 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:00:53.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:53.478154 15584 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:00:53.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:53.478706 15584 kserver.cc:163] Server-wide thread pool size limit: 3276
20:00:53.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:53.482182 15584 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:00:53.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:53.487820 15584 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:00:53.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:53.488090 15584 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:00:53.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:53.488411 15584 ts_tablet_manager.cc:616] Registered 0 tablets
20:00:53.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:53.488631 15584 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:00:53.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:53.714679 15584 rpc_server.cc:307] RPC server started. Bound to: 127.14.242.65:39413
20:00:53.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:53.714701 15737 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.14.242.65:39413 every 8 connection(s)
20:00:53.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:53.718461 15584 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-0/data/info.pb
20:00:53.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:53.720108 15305 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_muOGM/build/tsan/bin/kudu as pid 15584
20:00:53.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:53.720614 15305 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-0/wal/instance
20:00:53.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:53.731110 15305 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_muOGM/build/tsan/bin/kudu
20:00:53.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_muOGM/build/tsan/bin/kudu
20:00:53.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-1/wal
20:00:53.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-1/data
20:00:53.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:00:53.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:00:53.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:00:53.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:00:53.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:00:53.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:00:53.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-1/logs
20:00:53.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-1/data/info.pb
20:00:53.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:00:53.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:00:53.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:00:53.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:00:53.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:00:53.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:00:53.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:00:53.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:00:53.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:00:53.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:00:53.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.242.66:0
20:00:53.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.14.242.66
20:00:53.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.242.66
20:00:53.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:00:53.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.14.242.126:44505,127.14.242.125:42633,127.14.242.124:32797
20:00:53.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.242.84:46193
20:00:53.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:00:53.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:00:53.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:00:53.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:53.783309 15738 heartbeater.cc:344] Connected to a master server at 127.14.242.124:32797
20:00:53.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:53.784266 15738 heartbeater.cc:461] Registering TS with master...
20:00:53.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:53.785815 15738 heartbeater.cc:507] Master 127.14.242.124:32797 requested a full tablet report, sending...
20:00:53.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:53.789989 15505 ts_manager.cc:194] Registered new tserver with Master: a23ba18e795745cf998aaf04a40856cb (127.14.242.65:39413)
20:00:53.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:53.793242 15505 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.14.242.65:36493
20:00:53.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:53.816766 15739 heartbeater.cc:344] Connected to a master server at 127.14.242.125:42633
20:00:53.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:53.817159 15739 heartbeater.cc:461] Registering TS with master...
20:00:53.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:53.817836 15739 heartbeater.cc:507] Master 127.14.242.125:42633 requested a full tablet report, sending...
20:00:53.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:53.821308 15425 ts_manager.cc:194] Registered new tserver with Master: a23ba18e795745cf998aaf04a40856cb (127.14.242.65:39413)
20:00:53.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:53.825398 15743 heartbeater.cc:344] Connected to a master server at 127.14.242.126:44505
20:00:53.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:53.825758 15743 heartbeater.cc:461] Registering TS with master...
20:00:53.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:53.826398 15743 heartbeater.cc:507] Master 127.14.242.126:44505 requested a full tablet report, sending...
20:00:53.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:53.829216 15350 ts_manager.cc:194] Registered new tserver with Master: a23ba18e795745cf998aaf04a40856cb (127.14.242.65:39413)
20:00:54.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:54.235596 15742 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:00:54.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:54.236332 15742 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:00:54.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:54.237056 15742 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:00:54.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:54.299268 15742 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:00:54.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:54.301033 15742 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.14.242.66
20:00:54.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:54.349238 15742 tablet_server_runner.cc:78] Tablet server non-default flags:
20:00:54.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-1/logs
20:00:54.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:00:54.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:00:54.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:00:54.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.242.84:46193
20:00:54.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:00:54.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:00:54.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-1/data
20:00:54.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-1/wal
20:00:54.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:00:54.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:00:54.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.242.66:0
20:00:54.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:00:54.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:00:54.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:00:54.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-1/data/info.pb
20:00:54.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.242.66
20:00:54.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:00:54.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.14.242.126:44505,127.14.242.125:42633,127.14.242.124:32797
20:00:54.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:00:54.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:00:54.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:00:54.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:00:54.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:00:54.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.14.242.66
20:00:54.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:00:54.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:00:54.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:00:54.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3b18f0139b1d21a072b7353269301fba16dca54e
20:00:54.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:00:54.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Sep 2026 19:43:26 UTC on e7f111948823
20:00:54.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14030
20:00:54.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:00:54.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:54.351131 15742 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:00:54.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:54.353482 15742 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:00:54.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:54.370115 15754 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:54.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:54.808275 15738 heartbeater.cc:499] Master 127.14.242.124:32797 was elected leader, sending a full tablet report...
20:00:55.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:54.371050 15755 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:55.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:55.676527 15742 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.308s user 0.002s sys 0.006s
20:00:55.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:55.676960 15742 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.308s user 0.002s sys 0.006s
20:00:55.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:55.678876 15742 server_base.cc:1061] running on GCE node
20:00:55.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:55.680697 15761 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:55.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:55.682193 15742 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:00:55.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:55.685075 15742 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:00:55.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:55.686767 15742 hybrid_clock.cc:648] HybridClock initialized: now 1790539255686715 us; error 39 us; skew 500 ppm
20:00:55.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:55.688215 15742 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:00:55.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:55.697783 15742 webserver.cc:533] Webserver started at http://127.14.242.66:36441/ using document root <none> and password file <none>
20:00:55.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:55.699435 15742 fs_manager.cc:362] Metadata directory not provided
20:00:55.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:55.699993 15742 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:00:55.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:55.700841 15742 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:00:55.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:55.707988 15742 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-1/data/instance:
20:00:55.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "da91e771a46d4daf823255b31e97a963"
20:00:55.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:00:55 on dist-test-slave-1555"
20:00:55.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:55.709689 15742 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-1/wal/instance:
20:00:55.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "da91e771a46d4daf823255b31e97a963"
20:00:55.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:00:55 on dist-test-slave-1555"
20:00:55.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:55.721170 15742 fs_manager.cc:696] Time spent creating directory manager: real 0.011s user 0.011s sys 0.000s
20:00:55.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:55.729200 15766 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:00:55.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:55.730718 15742 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
20:00:55.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:55.731161 15742 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-1/data,/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-1/wal
20:00:55.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "da91e771a46d4daf823255b31e97a963"
20:00:55.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:00:55 on dist-test-slave-1555"
20:00:55.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:55.731995 15742 fs_report.cc:389] FS layout report
20:00:55.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:00:55.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-1/wal
20:00:55.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-1/wal
20:00:55.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-1/data/data
20:00:55.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:00:55.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:00:55.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:00:55.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:00:55.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:00:55.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:00:55.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:00:55.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:00:55.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:00:55.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:00:55.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:00:55.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:55.832753 15742 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:00:55.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:55.834652 15742 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:00:55.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:55.835222 15742 kserver.cc:163] Server-wide thread pool size limit: 3276
20:00:55.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:55.838267 15742 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:00:55.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:55.843644 15742 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:00:55.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:55.843914 15742 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:00:55.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:55.844244 15742 ts_tablet_manager.cc:616] Registered 0 tablets
20:00:55.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:55.844492 15742 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:00:55.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:55.987696 15742 rpc_server.cc:307] RPC server started. Bound to: 127.14.242.66:34153
20:00:55.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:55.987809 15878 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.14.242.66:34153 every 8 connection(s)
20:00:55.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:55.990586 15742 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-1/data/info.pb
20:00:55.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:55.994995 15305 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_muOGM/build/tsan/bin/kudu as pid 15742
20:00:55.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:55.995630 15305 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-1/wal/instance
20:00:56.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:56.028545 15305 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_muOGM/build/tsan/bin/kudu
20:00:56.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_muOGM/build/tsan/bin/kudu
20:00:56.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-2/wal
20:00:56.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-2/data
20:00:56.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:00:56.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:00:56.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:00:56.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:00:56.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:00:56.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:00:56.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-2/logs
20:00:56.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-2/data/info.pb
20:00:56.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:00:56.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:00:56.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:00:56.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:00:56.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:00:56.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:00:56.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:00:56.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:00:56.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:00:56.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:00:56.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.242.67:0
20:00:56.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.14.242.67
20:00:56.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.242.67
20:00:56.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:00:56.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.14.242.126:44505,127.14.242.125:42633,127.14.242.124:32797
20:00:56.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.242.84:46193
20:00:56.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:00:56.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:00:56.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:00:56.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:56.074090 15879 heartbeater.cc:344] Connected to a master server at 127.14.242.124:32797
20:00:56.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:56.074662 15881 heartbeater.cc:344] Connected to a master server at 127.14.242.126:44505
20:00:56.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:56.075167 15881 heartbeater.cc:461] Registering TS with master...
20:00:56.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:56.074694 15879 heartbeater.cc:461] Registering TS with master...
20:00:56.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:56.075958 15880 heartbeater.cc:344] Connected to a master server at 127.14.242.125:42633
20:00:56.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:56.076320 15880 heartbeater.cc:461] Registering TS with master...
20:00:56.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:56.076733 15881 heartbeater.cc:507] Master 127.14.242.126:44505 requested a full tablet report, sending...
20:00:56.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:56.077657 15880 heartbeater.cc:507] Master 127.14.242.125:42633 requested a full tablet report, sending...
20:00:56.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:56.078338 15879 heartbeater.cc:507] Master 127.14.242.124:32797 requested a full tablet report, sending...
20:00:56.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:56.080634 15505 ts_manager.cc:194] Registered new tserver with Master: da91e771a46d4daf823255b31e97a963 (127.14.242.66:34153)
20:00:56.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:56.081126 15350 ts_manager.cc:194] Registered new tserver with Master: da91e771a46d4daf823255b31e97a963 (127.14.242.66:34153)
20:00:56.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:56.081554 15425 ts_manager.cc:194] Registered new tserver with Master: da91e771a46d4daf823255b31e97a963 (127.14.242.66:34153)
20:00:56.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:56.082357 15505 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.14.242.66:32957
20:00:56.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:56.380849 15889 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:00:56.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:56.381350 15889 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:00:56.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:56.381750 15889 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:00:56.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:56.417209 15889 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:00:56.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:56.417999 15889 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.14.242.67
20:00:56.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:56.463490 15889 tablet_server_runner.cc:78] Tablet server non-default flags:
20:00:56.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-2/logs
20:00:56.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:00:56.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:00:56.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:00:56.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.242.84:46193
20:00:56.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:00:56.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:00:56.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-2/data
20:00:56.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-2/wal
20:00:56.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:00:56.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:00:56.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.242.67:0
20:00:56.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:00:56.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:00:56.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:00:56.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-2/data/info.pb
20:00:56.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.242.67
20:00:56.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:00:56.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.14.242.126:44505,127.14.242.125:42633,127.14.242.124:32797
20:00:56.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:00:56.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:00:56.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:00:56.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:00:56.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:00:56.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.14.242.67
20:00:56.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:00:56.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:00:56.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:00:56.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3b18f0139b1d21a072b7353269301fba16dca54e
20:00:56.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:00:56.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Sep 2026 19:43:26 UTC on e7f111948823
20:00:56.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14030
20:00:56.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:00:56.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:56.464998 15889 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:00:56.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:56.467243 15889 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:00:56.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:56.482072 15895 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:57.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:57.085817 15879 heartbeater.cc:499] Master 127.14.242.124:32797 was elected leader, sending a full tablet report...
20:00:58.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:56.482725 15896 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:58.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:58.197892 15889 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.715s user 0.636s sys 1.048s
20:00:58.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:57.891032 15894 debug-util.cc:398] Leaking SignalData structure 0x7b0800034de0 after lost signal to thread 15889
20:00:58.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:58.198299 15889 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.715s user 0.636s sys 1.048s
20:00:58.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:58.200825 15898 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:58.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:00:58.202698 15897 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1715 milliseconds
20:00:58.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:58.202752 15889 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:00:58.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:58.203962 15889 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:00:58.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:58.206086 15889 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:00:58.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:58.207484 15889 hybrid_clock.cc:648] HybridClock initialized: now 1790539258207448 us; error 34 us; skew 500 ppm
20:00:58.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:58.208437 15889 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:00:58.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:58.214596 15889 webserver.cc:533] Webserver started at http://127.14.242.67:36323/ using document root <none> and password file <none>
20:00:58.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:58.215564 15889 fs_manager.cc:362] Metadata directory not provided
20:00:58.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:58.215816 15889 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:00:58.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:58.216379 15889 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:00:58.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:58.220624 15889 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-2/data/instance:
20:00:58.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "840bcd693161472aaef90608f0125d6e"
20:00:58.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:00:58 on dist-test-slave-1555"
20:00:58.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:58.221836 15889 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-2/wal/instance:
20:00:58.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "840bcd693161472aaef90608f0125d6e"
20:00:58.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:00:58 on dist-test-slave-1555"
20:00:58.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:58.229064 15889 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.000s
20:00:58.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:58.234750 15906 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:00:58.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:58.236017 15889 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.001s
20:00:58.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:58.236460 15889 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-2/data,/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-2/wal
20:00:58.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "840bcd693161472aaef90608f0125d6e"
20:00:58.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:00:58 on dist-test-slave-1555"
20:00:58.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:58.236804 15889 fs_report.cc:389] FS layout report
20:00:58.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:00:58.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-2/wal
20:00:58.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-2/wal
20:00:58.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-2/data/data
20:00:58.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:00:58.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:00:58.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:00:58.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:00:58.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:00:58.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:00:58.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:00:58.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:00:58.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:00:58.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:00:58.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:00:58.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:58.301736 15889 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:00:58.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:58.303244 15889 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:00:58.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:58.303663 15889 kserver.cc:163] Server-wide thread pool size limit: 3276
20:00:58.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:58.306308 15889 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:00:58.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:58.310808 15889 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:00:58.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:58.311017 15889 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:00:58.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:58.311208 15889 ts_tablet_manager.cc:616] Registered 0 tablets
20:00:58.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:58.311348 15889 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:00:58.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:58.452615 15889 rpc_server.cc:307] RPC server started. Bound to: 127.14.242.67:32879
20:00:58.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:58.452772 16018 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.14.242.67:32879 every 8 connection(s)
20:00:58.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:58.455349 15889 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-2/data/info.pb
20:00:58.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:58.462123 15305 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_muOGM/build/tsan/bin/kudu as pid 15889
20:00:58.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:58.462585 15305 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster5213791184726944434/ts-2/wal/instance
20:00:58.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:58.501891 16019 heartbeater.cc:344] Connected to a master server at 127.14.242.124:32797
20:00:58.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:58.502389 16019 heartbeater.cc:461] Registering TS with master...
20:00:58.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:58.503712 16019 heartbeater.cc:507] Master 127.14.242.124:32797 requested a full tablet report, sending...
20:00:58.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:58.503860 16020 heartbeater.cc:344] Connected to a master server at 127.14.242.125:42633
20:00:58.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:58.504197 16020 heartbeater.cc:461] Registering TS with master...
20:00:58.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:58.505081 16020 heartbeater.cc:507] Master 127.14.242.125:42633 requested a full tablet report, sending...
20:00:58.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:58.506515 15505 ts_manager.cc:194] Registered new tserver with Master: 840bcd693161472aaef90608f0125d6e (127.14.242.67:32879)
20:00:58.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:58.508491 15425 ts_manager.cc:194] Registered new tserver with Master: 840bcd693161472aaef90608f0125d6e (127.14.242.67:32879)
20:00:58.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:58.508885 15505 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.14.242.67:45799
20:00:58.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:58.509495 16021 heartbeater.cc:344] Connected to a master server at 127.14.242.126:44505
20:00:58.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:58.509810 16021 heartbeater.cc:461] Registering TS with master...
20:00:58.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:58.510918 16021 heartbeater.cc:507] Master 127.14.242.126:44505 requested a full tablet report, sending...
20:00:58.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:58.513239 15350 ts_manager.cc:194] Registered new tserver with Master: 840bcd693161472aaef90608f0125d6e (127.14.242.67:32879)
20:00:58.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:58.533499 15305 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
20:00:58.545 [DEBUG - main] (MiniKuduCluster.java:195) Response:
20:00:58.546 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
20:00:58.555 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.14.242.126"
port: 44505
}
bound_http_address {
host: "127.14.242.126"
port: 44971
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.14.242.125"
port: 42633
}
bound_http_address {
host: "127.14.242.125"
port: 37275
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.14.242.124"
port: 32797
}
bound_http_address {
host: "127.14.242.124"
port: 36617
}
}
}
20:00:58.559 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
20:00:58.561 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.14.242.65"
port: 39413
}
bound_http_address {
host: "127.14.242.65"
port: 33123
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.14.242.66"
port: 34153
}
bound_http_address {
host: "127.14.242.66"
port: 36441
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.14.242.67"
port: 32879
}
bound_http_address {
host: "127.14.242.67"
port: 36323
}
}
}
20:00:58.561 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
20:00:58.768 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
20:00:58.833 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
20:00:58.942 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task_muOGM/test-tmp
20:00:58.943 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
20:00:58.943 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task_muOGM/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
20:00:58.949 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: true
cluster_root: "/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
20:00:59.304 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:00:59.305 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
20:00:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:59.310089 16032 mini_oidc.cc:150] Starting JWKS server
20:00:59.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:59.313588 16032 webserver.cc:533] Webserver started at http://127.0.0.1:41913/ using document root <none> and password file <none>
20:00:59.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:59.314105 16032 mini_oidc.cc:186] Starting OIDC Discovery server
20:00:59.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:59.315999 16032 webserver.cc:533] Webserver started at http://127.0.0.1:33861/ using document root <none> and password file <none>
20:00:59.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
20:00:59.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/principal' for realm 'KRBTEST.COM',
20:00:59.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
20:00:59.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:00:59 dist-test-slave-1555 krb5kdc[16044](info): setting up network...
20:00:59.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
20:00:59.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:00:59 dist-test-slave-1555 krb5kdc[16044](info): set up 2 sockets
20:00:59.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:00:59 dist-test-slave-1555 krb5kdc[16044](info): commencing operation
20:00:59.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
20:00:59.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:00:59.513708 16019 heartbeater.cc:499] Master 127.14.242.124:32797 was elected leader, sending a full tablet report...
20:01:01.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:01.495980 16032 mini_kdc.cc:121] Time spent starting KDC: real 2.141s user 0.005s sys 0.016s
20:01:01.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
20:01:01.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:01:01.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
20:01:01.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
20:01:01.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:01:01.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
20:01:01.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
20:01:01.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:01:01.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
20:01:01.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:01:01.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/test-user.keytab.
20:01:01.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/test-user.keytab.
20:01:01.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:01 dist-test-slave-1555 krb5kdc[16044](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539261, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:01.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:01:01.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T20:01:01Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
20:01:01.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T20:01:01Z Disabled control of system clock
20:01:01.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.15.168.62@KRBTEST.COM; defaulting to no policy
20:01:01.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:01:01.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.15.168.62@KRBTEST.COM" created.
20:01:01.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:01:01.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.15.168.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab.
20:01:01.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.15.168.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab.
20:01:01.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.15.168.62@KRBTEST.COM; defaulting to no policy
20:01:01.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:01:01.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.15.168.62@KRBTEST.COM" created.
20:01:01.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:01:01.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.15.168.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab.
20:01:01.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.15.168.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab.
20:01:01.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:01.783047 16032 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_muOGM/build/tsan/bin/kudu
20:01:01.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_muOGM/build/tsan/bin/kudu
20:01:01.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/wal
20:01:01.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/data
20:01:01.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:01:01.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:01:01.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:01:01.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:01.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:01.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:01.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/logs
20:01:01.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/data/info.pb
20:01:01.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:01.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:01:01.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:01:01.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:01:01.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:01:01.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:01.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:01.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:01.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:01:01.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:01:01.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:01.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:01.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.168.62:38531
20:01:01.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.168.62
20:01:01.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:01:01.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab
20:01:01.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.168.62
20:01:01.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:01.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:01:01.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:01:01.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:01:01.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.168.20:32967
20:01:01.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:01.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:01.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:01.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:01.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.168.62:38531,127.15.168.61:32863,127.15.168.60:45671
20:01:01.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:01.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41913/jwks.json
20:01:01.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:01:01.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:01:02.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:02.124550 16061 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:01:02.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:02.125109 16061 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:01:02.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:02.125710 16061 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:01:02.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:02.160184 16061 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:01:02.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:02.160430 16061 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:01:02.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:02.160693 16061 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:01:02.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:02.160893 16061 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:01:02.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:02.161401 16061 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:01:02.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:02.161631 16061 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:01:02.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:02.161773 16061 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:41913/jwks.json
20:01:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:02.202608 16061 master_runner.cc:387] Master server non-default flags:
20:01:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/logs
20:01:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:01:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.168.20:32967
20:01:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/data
20:01:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/wal
20:01:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:01:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.168.62:38531,127.15.168.61:32863,127.15.168.60:45671
20:01:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab
20:01:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.168.62
20:01:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.168.62:38531
20:01:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:01:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41913/jwks.json
20:01:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:01:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:01:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/data/info.pb
20:01:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.168.62
20:01:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:01:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:01:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:01:02.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:01:02.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:02.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:01:02.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:01:02.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:01:02.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3b18f0139b1d21a072b7353269301fba16dca54e
20:01:02.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:01:02.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Sep 2026 19:43:26 UTC on e7f111948823
20:01:02.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14030
20:01:02.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:01:02.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:02.203907 16061 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:01:02.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:02.205134 16061 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:01:02.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:02.218161 16067 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:04.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:03.620103 16066 debug-util.cc:398] Leaking SignalData structure 0x7b0800037cc0 after lost signal to thread 16061
20:01:04.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:04.049748 16066 kernel_stack_watchdog.cc:198] Thread 16061 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 399ms:
20:01:04.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:01:04.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:01:04.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:01:04.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:01:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:02.218695 16068 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:04.050595 16061 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.833s user 0.676s sys 1.129s
20:01:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:04.051501 16061 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.834s user 0.677s sys 1.129s
20:01:04.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:04.053009 16070 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:04.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:04.055332 16069 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1832 milliseconds
20:01:04.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:04.055362 16061 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:01:04.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:04.056658 16061 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:01:04.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:04.059019 16061 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:01:04.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:04.060361 16061 hybrid_clock.cc:648] HybridClock initialized: now 1790539264060327 us; error 43 us; skew 500 ppm
20:01:04.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:04.061137 16061 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:01:04.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:04 dist-test-slave-1555 krb5kdc[16044](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539264, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:04.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:04.067363 16061 init.cc:377] Logged in from keytab as kudu/127.15.168.62@KRBTEST.COM (short username kudu)
20:01:04.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:04.072814 16061 webserver.cc:533] Webserver started at http://127.15.168.62:41983/ using document root <none> and password file <none>
20:01:04.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:04.073704 16061 fs_manager.cc:362] Metadata directory not provided
20:01:04.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:04.073985 16061 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:01:04.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:04.074393 16061 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:01:04.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:04.078620 16061 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/data/instance:
20:01:04.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "95585e2dbd5a475f9466ce816f5fffff"
20:01:04.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:01:04 on dist-test-slave-1555"
20:01:04.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:04.079674 16061 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/wal/instance:
20:01:04.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "95585e2dbd5a475f9466ce816f5fffff"
20:01:04.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:01:04 on dist-test-slave-1555"
20:01:04.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:04.087427 16061 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.001s
20:01:04.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:04.093261 16078 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:01:04.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:04.094695 16061 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
20:01:04.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:04.095008 16061 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/data,/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/wal
20:01:04.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "95585e2dbd5a475f9466ce816f5fffff"
20:01:04.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:01:04 on dist-test-slave-1555"
20:01:04.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:04.095306 16061 fs_report.cc:389] FS layout report
20:01:04.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:01:04.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/wal
20:01:04.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/wal
20:01:04.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/data/data
20:01:04.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:01:04.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:01:04.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:01:04.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:01:04.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:01:04.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:01:04.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:01:04.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:01:04.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:01:04.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:01:04.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:01:04.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:04.156638 16061 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:01:04.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:04.167282 16061 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:01:04.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:04.167712 16061 kserver.cc:163] Server-wide thread pool size limit: 3276
20:01:04.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:04.246519 16061 rpc_server.cc:307] RPC server started. Bound to: 127.15.168.62:38531
20:01:04.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:04.246613 16130 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.168.62:38531 every 8 connection(s)
20:01:04.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:04.249619 16061 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/data/info.pb
20:01:04.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:04.255631 16131 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:01:04.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:04.259639 16032 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_muOGM/build/tsan/bin/kudu as pid 16061
20:01:04.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:04.260190 16032 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/wal/instance
20:01:04.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:04.270740 16131 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } has no permanent_uuid. Determining permanent_uuid...
20:01:04.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.15.168.61@KRBTEST.COM; defaulting to no policy
20:01:04.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:01:04.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.15.168.61@KRBTEST.COM" created.
20:01:04.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:04 dist-test-slave-1555 krb5kdc[16044](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539264, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.62@KRBTEST.COM for kudu/127.15.168.62@KRBTEST.COM
20:01:04.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:04.315068 16131 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } has no permanent_uuid. Determining permanent_uuid...
20:01:04.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:04.317478 16082 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.168.61:32863: connect: Connection refused (error 111)
20:01:04.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:01:04.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.15.168.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab.
20:01:04.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.15.168.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab.
20:01:04.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:04.324234 16131 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.168.61:32863: Network error: Client connection negotiation failed: client connection to 127.15.168.61:32863: connect: Connection refused (error 111)
20:01:04.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.15.168.61@KRBTEST.COM; defaulting to no policy
20:01:04.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:01:04.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.15.168.61@KRBTEST.COM" created.
20:01:04.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:01:04.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.15.168.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab.
20:01:04.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.15.168.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab.
20:01:04.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:04.360844 16032 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_muOGM/build/tsan/bin/kudu
20:01:04.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_muOGM/build/tsan/bin/kudu
20:01:04.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/wal
20:01:04.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/data
20:01:04.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:01:04.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:01:04.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:01:04.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:04.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:04.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:04.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/logs
20:01:04.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/data/info.pb
20:01:04.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:04.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:01:04.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:01:04.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:01:04.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:01:04.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:04.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:04.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:04.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:01:04.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:01:04.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:04.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:04.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.168.61:32863
20:01:04.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.168.61
20:01:04.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:01:04.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab
20:01:04.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.168.61
20:01:04.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:04.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:01:04.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:01:04.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:01:04.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.168.20:32967
20:01:04.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:04.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:04.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:04.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:04.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.168.62:38531,127.15.168.61:32863,127.15.168.60:45671
20:01:04.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:04.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41913/jwks.json
20:01:04.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:01:04.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:01:04.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:04.373852 16131 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } attempt: 1
20:01:04.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:04.379163 16131 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.168.61:32863: Network error: Client connection negotiation failed: client connection to 127.15.168.61:32863: connect: Connection refused (error 111)
20:01:04.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:04.447625 16131 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } attempt: 2
20:01:04.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:04.453329 16131 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.168.61:32863: Network error: Client connection negotiation failed: client connection to 127.15.168.61:32863: connect: Connection refused (error 111)
20:01:04.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:04.544754 16131 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } attempt: 3
20:01:04.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:04.548933 16131 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.168.61:32863: Network error: Client connection negotiation failed: client connection to 127.15.168.61:32863: connect: Connection refused (error 111)
20:01:04.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:04.692431 16131 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } attempt: 4
20:01:04.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:04.696792 16131 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.168.61:32863: Network error: Client connection negotiation failed: client connection to 127.15.168.61:32863: connect: Connection refused (error 111)
20:01:04.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:04.703788 16140 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:01:04.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:04.704396 16140 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:01:04.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:04.705132 16140 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:01:04.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:04.739497 16140 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:01:04.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:04.739835 16140 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:01:04.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:04.740183 16140 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:01:04.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:04.740406 16140 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:01:04.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:04.740608 16140 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:01:04.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:04.740794 16140 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:01:04.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:04.740933 16140 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:41913/jwks.json
20:01:04.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:04.780417 16140 master_runner.cc:387] Master server non-default flags:
20:01:04.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/logs
20:01:04.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:04.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:01:04.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:04.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.168.20:32967
20:01:04.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:04.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:04.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/data
20:01:04.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/wal
20:01:04.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:01:04.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:04.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.168.62:38531,127.15.168.61:32863,127.15.168.60:45671
20:01:04.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab
20:01:04.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.168.61
20:01:04.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:04.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:04.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:04.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.168.61:32863
20:01:04.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:04.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:01:04.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:04.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41913/jwks.json
20:01:04.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:04.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:01:04.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:01:04.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:04.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:04.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/data/info.pb
20:01:04.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.168.61
20:01:04.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:01:04.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:01:04.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:04.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:01:04.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:01:04.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:04.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:01:04.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:01:04.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:01:04.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3b18f0139b1d21a072b7353269301fba16dca54e
20:01:04.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:01:04.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Sep 2026 19:43:26 UTC on e7f111948823
20:01:04.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14030
20:01:04.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:01:04.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:04.781787 16140 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:01:04.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:04.782750 16140 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:01:04.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:04.793917 16147 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:04.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:04.996245 16131 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } attempt: 5
20:01:05.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:05.004201 16131 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.168.61:32863: Network error: Client connection negotiation failed: client connection to 127.15.168.61:32863: connect: Connection refused (error 111)
20:01:05.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:05.551679 16131 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } attempt: 6
20:01:05.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:05.592427 16131 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.168.61:32863: Network error: Client connection negotiation failed: client connection to 127.15.168.61:32863: connect: Connection refused (error 111)
20:01:06.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:06.654331 16131 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } attempt: 7
20:01:06.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:06.198299 16146 debug-util.cc:398] Leaking SignalData structure 0x7b0800037cc0 after lost signal to thread 16140
20:01:06.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:06.680024 16146 kernel_stack_watchdog.cc:198] Thread 16140 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 401ms:
20:01:06.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:01:06.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:01:06.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:01:06.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:01:06.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:04.794467 16148 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:06.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:06.681646 16140 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.888s user 0.602s sys 1.204s
20:01:06.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:06.682222 16140 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.888s user 0.602s sys 1.204s
20:01:06.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:06.684437 16131 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.168.61:32863: Network error: Client connection negotiation failed: client connection to 127.15.168.61:32863: connect: Connection refused (error 111)
20:01:06.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:06.687116 16149 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1886 milliseconds
20:01:06.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:06.689081 16150 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:06.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:06.689137 16140 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:01:06.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:06.690450 16140 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:01:06.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:06.692904 16140 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:01:06.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:06.694254 16140 hybrid_clock.cc:648] HybridClock initialized: now 1790539266694200 us; error 48 us; skew 500 ppm
20:01:06.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:06.695029 16140 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:01:06.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:06 dist-test-slave-1555 krb5kdc[16044](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539266, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:06.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:06.701089 16140 init.cc:377] Logged in from keytab as kudu/127.15.168.61@KRBTEST.COM (short username kudu)
20:01:06.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:06.705574 16140 webserver.cc:533] Webserver started at http://127.15.168.61:35057/ using document root <none> and password file <none>
20:01:06.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:06.706439 16140 fs_manager.cc:362] Metadata directory not provided
20:01:06.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:06.706619 16140 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:01:06.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:06.707018 16140 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:01:06.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:06.711647 16140 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/data/instance:
20:01:06.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4b00615a9e9244d2a54e01564420e235"
20:01:06.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:01:06 on dist-test-slave-1555"
20:01:06.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:06.713043 16140 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/wal/instance:
20:01:06.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4b00615a9e9244d2a54e01564420e235"
20:01:06.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:01:06 on dist-test-slave-1555"
20:01:06.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:06.722651 16140 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.006s sys 0.003s
20:01:06.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:06.729635 16164 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:01:06.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:06.730984 16140 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
20:01:06.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:06.731271 16140 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/data,/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/wal
20:01:06.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4b00615a9e9244d2a54e01564420e235"
20:01:06.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:01:06 on dist-test-slave-1555"
20:01:06.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:06.731701 16140 fs_report.cc:389] FS layout report
20:01:06.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:01:06.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/wal
20:01:06.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/wal
20:01:06.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/data/data
20:01:06.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:01:06.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:01:06.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:01:06.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:01:06.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:01:06.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:01:06.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:01:06.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:01:06.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:01:06.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:01:06.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:01:06.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:06.786902 16140 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:01:06.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:06.792577 16140 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:01:06.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:06.793044 16140 kserver.cc:163] Server-wide thread pool size limit: 3276
20:01:06.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:06.869211 16140 rpc_server.cc:307] RPC server started. Bound to: 127.15.168.61:32863
20:01:06.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:06.869283 16216 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.168.61:32863 every 8 connection(s)
20:01:06.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:06.872185 16140 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/data/info.pb
20:01:06.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:06.874297 16032 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_muOGM/build/tsan/bin/kudu as pid 16140
20:01:06.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:06.874883 16032 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/wal/instance
20:01:06.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:06.880226 16217 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:01:06.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.15.168.60@KRBTEST.COM; defaulting to no policy
20:01:06.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:01:06.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.15.168.60@KRBTEST.COM" created.
20:01:06.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:06.899075 16217 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } has no permanent_uuid. Determining permanent_uuid...
20:01:06.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:06 dist-test-slave-1555 krb5kdc[16044](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539266, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.61@KRBTEST.COM for kudu/127.15.168.62@KRBTEST.COM
20:01:06.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:01:06.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.15.168.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab.
20:01:06.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.15.168.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab.
20:01:06.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:06.943003 16217 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } has no permanent_uuid. Determining permanent_uuid...
20:01:06.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.15.168.60@KRBTEST.COM; defaulting to no policy
20:01:06.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:01:06.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.15.168.60@KRBTEST.COM" created.
20:01:06.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:06 dist-test-slave-1555 krb5kdc[16044](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539266, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.61@KRBTEST.COM for kudu/127.15.168.61@KRBTEST.COM
20:01:06.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:06.971124 16217 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } has no permanent_uuid. Determining permanent_uuid...
20:01:06.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:06.974025 16167 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.168.60:45671: connect: Connection refused (error 111)
20:01:06.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:06.978199 16217 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.168.60:45671: Network error: Client connection negotiation failed: client connection to 127.15.168.60:45671: connect: Connection refused (error 111)
20:01:06.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:01:06.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.15.168.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab.
20:01:06.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.15.168.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab.
20:01:06.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:06.981864 16032 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_muOGM/build/tsan/bin/kudu
20:01:06.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_muOGM/build/tsan/bin/kudu
20:01:06.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/wal
20:01:06.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/data
20:01:06.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:01:06.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:01:06.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:01:06.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:06.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:06.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:06.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/logs
20:01:06.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/data/info.pb
20:01:06.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:06.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:01:06.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:01:06.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:01:06.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:01:06.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:06.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:06.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:06.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:01:06.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:01:06.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:06.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:06.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.168.60:45671
20:01:06.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.168.60
20:01:06.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:01:06.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab
20:01:06.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.168.60
20:01:06.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:06.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:01:06.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:01:06.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:01:06.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.168.20:32967
20:01:06.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:06.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:06.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:06.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:06.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.168.62:38531,127.15.168.61:32863,127.15.168.60:45671
20:01:06.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:06.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41913/jwks.json
20:01:06.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:01:06.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:01:07.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:07.027787 16217 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } attempt: 1
20:01:07.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:07.031994 16217 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.168.60:45671: Network error: Client connection negotiation failed: client connection to 127.15.168.60:45671: connect: Connection refused (error 111)
20:01:07.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:07.100461 16217 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } attempt: 2
20:01:07.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:07.104770 16217 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.168.60:45671: Network error: Client connection negotiation failed: client connection to 127.15.168.60:45671: connect: Connection refused (error 111)
20:01:07.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:07.196244 16217 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } attempt: 3
20:01:07.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:07.200549 16217 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.168.60:45671: Network error: Client connection negotiation failed: client connection to 127.15.168.60:45671: connect: Connection refused (error 111)
20:01:07.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:07.338258 16227 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:01:07.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:07.338824 16227 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:01:07.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:07.339597 16227 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:01:07.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:07.344038 16217 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } attempt: 4
20:01:07.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:07.348570 16217 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.168.60:45671: Network error: Client connection negotiation failed: client connection to 127.15.168.60:45671: connect: Connection refused (error 111)
20:01:07.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:07.375082 16227 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:01:07.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:07.375316 16227 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:01:07.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:07.375584 16227 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:01:07.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:07.375783 16227 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:01:07.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:07.375984 16227 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:01:07.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:07.376219 16227 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:01:07.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:07.376379 16227 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:41913/jwks.json
20:01:07.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:07.417098 16227 master_runner.cc:387] Master server non-default flags:
20:01:07.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/logs
20:01:07.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:07.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:01:07.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:07.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.168.20:32967
20:01:07.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:07.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:07.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/data
20:01:07.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/wal
20:01:07.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:01:07.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:07.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.168.62:38531,127.15.168.61:32863,127.15.168.60:45671
20:01:07.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab
20:01:07.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.168.60
20:01:07.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:07.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:07.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:07.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.168.60:45671
20:01:07.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:07.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:01:07.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:07.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41913/jwks.json
20:01:07.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:07.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:01:07.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:01:07.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:07.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:07.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/data/info.pb
20:01:07.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.168.60
20:01:07.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:01:07.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:01:07.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:07.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:01:07.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:01:07.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:07.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:01:07.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:01:07.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:01:07.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3b18f0139b1d21a072b7353269301fba16dca54e
20:01:07.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:01:07.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Sep 2026 19:43:26 UTC on e7f111948823
20:01:07.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14030
20:01:07.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:01:07.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:07.418322 16227 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:01:07.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:07.419322 16227 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:01:07.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:07.430342 16233 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:07.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:07.648015 16217 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } attempt: 5
20:01:07.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:07.655252 16217 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.168.60:45671: Network error: Client connection negotiation failed: client connection to 127.15.168.60:45671: connect: Connection refused (error 111)
20:01:08.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:08.202832 16217 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } attempt: 6
20:01:08.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:08.216117 16217 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.168.60:45671: Network error: Client connection negotiation failed: client connection to 127.15.168.60:45671: connect: Connection refused (error 111)
20:01:08.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:08.774830 16131 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } attempt: 8
20:01:08.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:08 dist-test-slave-1555 krb5kdc[16044](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539264, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.62@KRBTEST.COM for kudu/127.15.168.61@KRBTEST.COM
20:01:08.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:07.431766 16234 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:08.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:08.812430 16236 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:08.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:08.816395 16235 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1379 milliseconds
20:01:08.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:08.816543 16227 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:01:08.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:08.817945 16227 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:01:08.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:08.820127 16131 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } has no permanent_uuid. Determining permanent_uuid...
20:01:08.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:08.821540 16227 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:01:08.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:08.823184 16227 hybrid_clock.cc:648] HybridClock initialized: now 1790539268823103 us; error 77 us; skew 500 ppm
20:01:08.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:08.824617 16227 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:01:08.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:08 dist-test-slave-1555 krb5kdc[16044](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539268, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:08.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:08.830729 16131 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.168.60:45671: Network error: Client connection negotiation failed: client connection to 127.15.168.60:45671: connect: Connection refused (error 111)
20:01:08.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:08.831866 16227 init.cc:377] Logged in from keytab as kudu/127.15.168.60@KRBTEST.COM (short username kudu)
20:01:08.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:08.837275 16227 webserver.cc:533] Webserver started at http://127.15.168.60:33645/ using document root <none> and password file <none>
20:01:08.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:08.838306 16227 fs_manager.cc:362] Metadata directory not provided
20:01:08.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:08.838559 16227 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:01:08.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:08.839159 16227 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:01:08.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:08.843644 16227 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/data/instance:
20:01:08.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "33ab2c6f07d14efa88f25702047f032c"
20:01:08.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:01:08 on dist-test-slave-1555"
20:01:08.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:08.844830 16227 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/wal/instance:
20:01:08.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "33ab2c6f07d14efa88f25702047f032c"
20:01:08.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:01:08 on dist-test-slave-1555"
20:01:08.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:08.852994 16227 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.000s
20:01:08.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:08.859468 16250 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:01:08.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:08.860776 16227 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.000s
20:01:08.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:08.861153 16227 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/data,/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/wal
20:01:08.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "33ab2c6f07d14efa88f25702047f032c"
20:01:08.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:01:08 on dist-test-slave-1555"
20:01:08.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:08.861572 16227 fs_report.cc:389] FS layout report
20:01:08.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:01:08.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/wal
20:01:08.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/wal
20:01:08.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/data/data
20:01:08.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:01:08.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:01:08.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:01:08.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:01:08.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:01:08.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:01:08.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:01:08.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:01:08.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:01:08.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:01:08.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:01:08.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:08.896276 16131 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } attempt: 1
20:01:08.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:08.901507 16131 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.168.60:45671: Network error: Client connection negotiation failed: client connection to 127.15.168.60:45671: connect: Connection refused (error 111)
20:01:08.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:08.916445 16227 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:01:08.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:08.922456 16227 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:01:08.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:08.922871 16227 kserver.cc:163] Server-wide thread pool size limit: 3276
20:01:08.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:08.955010 16131 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } attempt: 2
20:01:08.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:08.961776 16131 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.15.168.60:45671: Network error: Client connection negotiation failed: client connection to 127.15.168.60:45671: connect: Connection refused (error 111)
20:01:09.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.003695 16227 rpc_server.cc:307] RPC server started. Bound to: 127.15.168.60:45671
20:01:09.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.003754 16301 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.168.60:45671 every 8 connection(s)
20:01:09.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.007421 16227 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/data/info.pb
20:01:09.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.009366 16032 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_muOGM/build/tsan/bin/kudu as pid 16227
20:01:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.009891 16032 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/wal/instance
20:01:09.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.015494 16302 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:01:09.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.032672 16302 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } has no permanent_uuid. Determining permanent_uuid...
20:01:09.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.038357 16131 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } attempt: 3
20:01:09.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:09 dist-test-slave-1555 krb5kdc[16044](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539261, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.15.168.62@KRBTEST.COM
20:01:09.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:09 dist-test-slave-1555 krb5kdc[16044](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539268, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.60@KRBTEST.COM for kudu/127.15.168.62@KRBTEST.COM
20:01:09.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:09 dist-test-slave-1555 krb5kdc[16044](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539264, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.62@KRBTEST.COM for kudu/127.15.168.60@KRBTEST.COM
20:01:09.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:09 dist-test-slave-1555 krb5kdc[16044](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539261, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.15.168.61@KRBTEST.COM
20:01:09.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.077303 16302 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } has no permanent_uuid. Determining permanent_uuid...
20:01:09.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.088217 16131 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff: Bootstrap starting.
20:01:09.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:09 dist-test-slave-1555 krb5kdc[16044](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539268, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.60@KRBTEST.COM for kudu/127.15.168.61@KRBTEST.COM
20:01:09.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.097262 16131 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff: Neither blocks nor log segments found. Creating new log.
20:01:09.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.100303 16131 log.cc:826] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff: Log is configured to *not* fsync() on all Append() calls
20:01:09.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:09 dist-test-slave-1555 krb5kdc[16044](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539261, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.15.168.60@KRBTEST.COM
20:01:09.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.106038 16302 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } has no permanent_uuid. Determining permanent_uuid...
20:01:09.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.108642 16131 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff: No bootstrap required, opened a new log
20:01:09.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:09 dist-test-slave-1555 krb5kdc[16044](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539268, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.60@KRBTEST.COM for kudu/127.15.168.60@KRBTEST.COM
20:01:09.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.137724 16131 raft_consensus.cc:359] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } }
20:01:09.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.138991 16131 raft_consensus.cc:385] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:01:09.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.139797 16131 raft_consensus.cc:740] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 95585e2dbd5a475f9466ce816f5fffff, State: Initialized, Role: FOLLOWER
20:01:09.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.141129 16131 consensus_queue.cc:260] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } }
20:01:09.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.143091 16302 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c: Bootstrap starting.
20:01:09.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.148670 16131 sys_catalog.cc:565] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [sys.catalog]: configured and running, proceeding with master startup.
20:01:09.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.150520 16302 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c: Neither blocks nor log segments found. Creating new log.
20:01:09.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.153491 16302 log.cc:826] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c: Log is configured to *not* fsync() on all Append() calls
20:01:09.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.155174 16312 sys_catalog.cc:455] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } } }
20:01:09.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.156200 16312 sys_catalog.cc:458] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [sys.catalog]: This master's current role is: FOLLOWER
20:01:09.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.166069 16302 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c: No bootstrap required, opened a new log
20:01:09.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:09.180120 16084 tablet.cc:2433] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:01:09.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:09.196808 16324 catalog_manager.cc:1607] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:01:09.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:09.197604 16324 catalog_manager.cc:896] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:01:09.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.197446 16302 raft_consensus.cc:359] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } }
20:01:09.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.198583 16302 raft_consensus.cc:385] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:01:09.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.198892 16302 raft_consensus.cc:740] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 33ab2c6f07d14efa88f25702047f032c, State: Initialized, Role: FOLLOWER
20:01:09.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.199887 16302 consensus_queue.cc:260] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } }
20:01:09.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.212111 16302 sys_catalog.cc:565] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [sys.catalog]: configured and running, proceeding with master startup.
20:01:09.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.208714 16326 sys_catalog.cc:455] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } } }
20:01:09.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.217178 16326 sys_catalog.cc:458] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [sys.catalog]: This master's current role is: FOLLOWER
20:01:09.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.217404 16327 raft_consensus.cc:493] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:01:09.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.218250 16327 raft_consensus.cc:515] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } }
20:01:09.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.221199 16327 leader_election.cc:290] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 95585e2dbd5a475f9466ce816f5fffff (127.15.168.62:38531), 4b00615a9e9244d2a54e01564420e235 (127.15.168.61:32863)
20:01:09.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.223712 16106 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "33ab2c6f07d14efa88f25702047f032c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "95585e2dbd5a475f9466ce816f5fffff" is_pre_election: true
20:01:09.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.224666 16106 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 33ab2c6f07d14efa88f25702047f032c in term 0.
20:01:09.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.226564 16254 leader_election.cc:304] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 33ab2c6f07d14efa88f25702047f032c, 95585e2dbd5a475f9466ce816f5fffff; no voters:
20:01:09.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.229249 16327 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [term 0 FOLLOWER]: Leader pre-election won for term 1
20:01:09.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.229625 16327 raft_consensus.cc:493] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:01:09.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.229964 16327 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [term 0 FOLLOWER]: Advancing to term 1
20:01:09.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.194774 16061 master_runner.cc:187] Error getting master registration for 127.15.168.61:32863: OK, instance_id { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" instance_seqno: 1790539266849024 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:01:09.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.249881 16061 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:01:09.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.261518 16327 raft_consensus.cc:515] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } }
20:01:09.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.263453 16327 leader_election.cc:290] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [CANDIDATE]: Term 1 election: Requested vote from peers 95585e2dbd5a475f9466ce816f5fffff (127.15.168.62:38531), 4b00615a9e9244d2a54e01564420e235 (127.15.168.61:32863)
20:01:09.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.265962 16106 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "33ab2c6f07d14efa88f25702047f032c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "95585e2dbd5a475f9466ce816f5fffff"
20:01:09.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.266548 16106 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [term 0 FOLLOWER]: Advancing to term 1
20:01:09.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.276692 16217 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } attempt: 7
20:01:09.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:09 dist-test-slave-1555 krb5kdc[16044](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539266, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.61@KRBTEST.COM for kudu/127.15.168.60@KRBTEST.COM
20:01:09.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.312736 16340 catalog_manager.cc:818] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
20:01:09.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.343639 16217 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235: Bootstrap starting.
20:01:09.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.352573 16217 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235: Neither blocks nor log segments found. Creating new log.
20:01:09.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.355159 16217 log.cc:826] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235: Log is configured to *not* fsync() on all Append() calls
20:01:09.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.362159 16106 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 33ab2c6f07d14efa88f25702047f032c in term 1.
20:01:09.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.363912 16254 leader_election.cc:304] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 33ab2c6f07d14efa88f25702047f032c, 95585e2dbd5a475f9466ce816f5fffff; no voters:
20:01:09.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.364861 16327 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [term 1 FOLLOWER]: Leader election won for term 1
20:01:09.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.365341 16327 raft_consensus.cc:697] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [term 1 LEADER]: Becoming Leader. State: Replica: 33ab2c6f07d14efa88f25702047f032c, State: Running, Role: LEADER
20:01:09.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.366278 16327 consensus_queue.cc:237] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } }
20:01:09.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.379849 16217 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235: No bootstrap required, opened a new log
20:01:09.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.391119 16326 sys_catalog.cc:455] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [sys.catalog]: SysCatalogTable state changed. Reason: New leader 33ab2c6f07d14efa88f25702047f032c. Latest consensus state: current_term: 1 leader_uuid: "33ab2c6f07d14efa88f25702047f032c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } } }
20:01:09.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.392465 16326 sys_catalog.cc:458] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [sys.catalog]: This master's current role is: LEADER
20:01:09.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.409945 16343 catalog_manager.cc:1524] Loading table and tablet metadata into memory...
20:01:09.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.442289 16217 raft_consensus.cc:359] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } }
20:01:09.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.443614 16217 raft_consensus.cc:385] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:01:09.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:09.444095 16256 tablet.cc:2433] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:01:09.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.444195 16217 raft_consensus.cc:740] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4b00615a9e9244d2a54e01564420e235, State: Initialized, Role: FOLLOWER
20:01:09.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.445652 16217 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } }
20:01:09.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.455896 16345 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } } }
20:01:09.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.389154 16227 master_runner.cc:187] Error getting master registration for 127.15.168.61:32863: OK, instance_id { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" instance_seqno: 1790539266849024 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:01:09.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.458734 16227 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:01:09.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.460034 16345 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [sys.catalog]: This master's current role is: FOLLOWER
20:01:09.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.460201 16343 catalog_manager.cc:1533] Initializing Kudu cluster ID...
20:01:09.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.459656 16217 sys_catalog.cc:565] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [sys.catalog]: configured and running, proceeding with master startup.
20:01:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.464476 16192 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "33ab2c6f07d14efa88f25702047f032c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4b00615a9e9244d2a54e01564420e235" is_pre_election: true
20:01:09.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.466272 16192 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 33ab2c6f07d14efa88f25702047f032c in term 0.
20:01:09.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.465404 16191 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "33ab2c6f07d14efa88f25702047f032c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4b00615a9e9244d2a54e01564420e235"
20:01:09.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.468613 16191 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [term 0 FOLLOWER]: Advancing to term 1
20:01:09.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.534106 16106 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [term 1 FOLLOWER]: Refusing update from remote peer 33ab2c6f07d14efa88f25702047f032c: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:01:09.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.542742 16326 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [LEADER]: Connected to new peer: Peer: permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:01:09.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:09.551261 16170 tablet.cc:2433] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:01:09.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.583926 16191 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 33ab2c6f07d14efa88f25702047f032c in term 1.
20:01:09.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.588872 16192 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [term 1 FOLLOWER]: Refusing update from remote peer 33ab2c6f07d14efa88f25702047f032c: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:01:09.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.595498 16312 sys_catalog.cc:455] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [sys.catalog]: SysCatalogTable state changed. Reason: New leader 33ab2c6f07d14efa88f25702047f032c. Latest consensus state: current_term: 1 leader_uuid: "33ab2c6f07d14efa88f25702047f032c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } } }
20:01:09.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.596230 16312 sys_catalog.cc:458] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [sys.catalog]: This master's current role is: FOLLOWER
20:01:09.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.605690 16326 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [LEADER]: Connected to new peer: Peer: permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:01:09.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.620366 16348 mvcc.cc:204] Tried to move back new op lower bound from 7334048847885889536 to 7334048847354523648. Current Snapshot: MvccSnapshot[applied={T|T < 7334048847885889536}]
20:01:09.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.632818 16312 sys_catalog.cc:455] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "33ab2c6f07d14efa88f25702047f032c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } } }
20:01:09.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.633530 16312 sys_catalog.cc:458] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [sys.catalog]: This master's current role is: FOLLOWER
20:01:09.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.643548 16327 sys_catalog.cc:455] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "33ab2c6f07d14efa88f25702047f032c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } } }
20:01:09.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.644346 16327 sys_catalog.cc:458] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [sys.catalog]: This master's current role is: LEADER
20:01:09.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.649935 16343 catalog_manager.cc:1396] Generated new cluster ID: d6c21a38473c4d4189831994cb48f2ce
20:01:09.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.650385 16343 catalog_manager.cc:1544] Initializing Kudu internal certificate authority...
20:01:09.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:09.652447 16359 catalog_manager.cc:1607] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:01:09.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:09.652957 16359 catalog_manager.cc:896] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:01:09.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.678247 16326 sys_catalog.cc:455] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "33ab2c6f07d14efa88f25702047f032c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } } }
20:01:09.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.679167 16326 sys_catalog.cc:458] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [sys.catalog]: This master's current role is: LEADER
20:01:09.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.680644 16345 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 33ab2c6f07d14efa88f25702047f032c. Latest consensus state: current_term: 1 leader_uuid: "33ab2c6f07d14efa88f25702047f032c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } } }
20:01:09.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.681331 16345 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [sys.catalog]: This master's current role is: FOLLOWER
20:01:09.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.709039 16343 catalog_manager.cc:1419] Generated new certificate authority record
20:01:09.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.711345 16343 catalog_manager.cc:1553] Loading token signing keys...
20:01:09.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.712477 16365 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "33ab2c6f07d14efa88f25702047f032c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } } }
20:01:09.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.713285 16365 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [sys.catalog]: This master's current role is: FOLLOWER
20:01:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.808240 16343 catalog_manager.cc:6219] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c: Generated new TSK 0
20:01:09.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:09.809821 16343 catalog_manager.cc:1563] Initializing in-progress tserver states...
20:01:09.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.15.168.1@KRBTEST.COM; defaulting to no policy
20:01:10.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:01:10.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.15.168.1@KRBTEST.COM" created.
20:01:10.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:10.204437 16324 catalog_manager.cc:1308] Loaded cluster ID: d6c21a38473c4d4189831994cb48f2ce
20:01:10.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:10.205046 16324 catalog_manager.cc:1601] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff: loading cluster ID for follower catalog manager: success
20:01:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:10.213222 16324 catalog_manager.cc:1623] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff: acquiring CA information for follower catalog manager: success
20:01:10.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:10.220835 16324 catalog_manager.cc:1651] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:01:10.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:10.665184 16359 catalog_manager.cc:1308] Loaded cluster ID: d6c21a38473c4d4189831994cb48f2ce
20:01:10.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:10.665939 16359 catalog_manager.cc:1601] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235: loading cluster ID for follower catalog manager: success
20:01:10.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:10.674508 16359 catalog_manager.cc:1623] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235: acquiring CA information for follower catalog manager: success
20:01:10.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:10.681115 16359 catalog_manager.cc:1651] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:01:11.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:01:11.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.15.168.1 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab.
20:01:11.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.15.168.1 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab.
20:01:11.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:11.878049 16032 mini_kdc.cc:268] Time spent creating service keytab for kudu/127.15.168.1: real 2.024s user 0.004s sys 0.012s
20:01:11.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.15.168.1@KRBTEST.COM; defaulting to no policy
20:01:11.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:01:11.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.15.168.1@KRBTEST.COM" created.
20:01:12.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:01:12.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.15.168.1 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab.
20:01:12.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.15.168.1 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab.
20:01:12.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:12.005609 16032 mini_kdc.cc:268] Time spent creating service keytab for HTTP/127.15.168.1: real 0.127s user 0.003s sys 0.011s
20:01:12.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:12.008021 16032 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_muOGM/build/tsan/bin/kudu
20:01:12.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_muOGM/build/tsan/bin/kudu
20:01:12.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-0/wal
20:01:12.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-0/data
20:01:12.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:01:12.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:01:12.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:01:12.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:12.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:12.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:12.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-0/logs
20:01:12.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-0/data/info.pb
20:01:12.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:12.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:01:12.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:01:12.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:01:12.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:01:12.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:12.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:12.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:12.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:01:12.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:01:12.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.168.1:0
20:01:12.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.15.168.1
20:01:12.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.168.1
20:01:12.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:01:12.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.15.168.62:38531,127.15.168.61:32863,127.15.168.60:45671
20:01:12.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab
20:01:12.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.168.1
20:01:12.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:12.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:01:12.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:01:12.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:01:12.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.168.20:32967
20:01:12.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:12.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:12.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:12.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:01:12.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:12.685622 16375 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:01:12.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:12.686401 16375 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:01:12.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:12.687032 16375 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:01:12.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:12.722493 16375 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:01:12.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:12.723889 16375 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.15.168.1
20:01:12.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:12.763230 16375 tablet_server_runner.cc:78] Tablet server non-default flags:
20:01:12.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-0/logs
20:01:12.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:12.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:01:12.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:12.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.168.20:32967
20:01:12.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:12.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:12.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-0/data
20:01:12.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-0/wal
20:01:12.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab
20:01:12.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.168.1
20:01:12.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:12.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:12.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.168.1:0
20:01:12.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:01:12.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:12.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:01:12.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:01:12.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:12.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:12.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-0/data/info.pb
20:01:12.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.168.1
20:01:12.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:01:12.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.15.168.62:38531,127.15.168.61:32863,127.15.168.60:45671
20:01:12.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:01:12.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:12.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:01:12.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:01:12.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:12.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.15.168.1
20:01:12.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:01:12.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:01:12.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:01:12.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3b18f0139b1d21a072b7353269301fba16dca54e
20:01:12.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:01:12.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Sep 2026 19:43:26 UTC on e7f111948823
20:01:12.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14030
20:01:12.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:01:12.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:12.764662 16375 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:01:12.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:12.765833 16375 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:01:12.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:12.782749 16383 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:14.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:14 dist-test-slave-1555 krb5kdc[16044](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539274, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:14.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:14.073501 16074 init.cc:295] Successfully reacquired a new kerberos TGT
20:01:14.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:14.189167 16381 debug-util.cc:398] Leaking SignalData structure 0x7b08000372a0 after lost signal to thread 16375
20:01:14.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:14.377004 16381 kernel_stack_watchdog.cc:198] Thread 16375 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 403ms:
20:01:14.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:01:14.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:01:14.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:01:14.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:01:14.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:12.782757 16382 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:14.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:14.378677 16375 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.596s user 0.490s sys 0.923s
20:01:14.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:14.379081 16375 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.596s user 0.490s sys 0.923s
20:01:14.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:14.380452 16385 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:14.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:14.383380 16384 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1594 milliseconds
20:01:14.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:14.383426 16375 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:01:14.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:14.384706 16375 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:01:14.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:14.387841 16375 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:01:14.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:14.389250 16375 hybrid_clock.cc:648] HybridClock initialized: now 1790539274389199 us; error 50 us; skew 500 ppm
20:01:14.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:14.390206 16375 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:01:14.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:14 dist-test-slave-1555 krb5kdc[16044](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539274, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.1@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:14.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:14.400802 16375 init.cc:377] Logged in from keytab as kudu/127.15.168.1@KRBTEST.COM (short username kudu)
20:01:14.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:14.414861 16375 webserver.cc:533] Webserver started at http://127.15.168.1:35773/ using document root <none> and password file <none>
20:01:14.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:14.416306 16375 fs_manager.cc:362] Metadata directory not provided
20:01:14.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:14.416555 16375 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:01:14.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:14.417263 16375 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:01:14.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:14.424639 16375 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-0/data/instance:
20:01:14.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cd0cbe577a5e4bce9d77779ab1e537e4"
20:01:14.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:01:14 on dist-test-slave-1555"
20:01:14.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:14.425863 16375 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-0/wal/instance:
20:01:14.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cd0cbe577a5e4bce9d77779ab1e537e4"
20:01:14.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:01:14 on dist-test-slave-1555"
20:01:14.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:14.435439 16375 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.010s sys 0.000s
20:01:14.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:14.443138 16394 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:01:14.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:14.444542 16375 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.007s sys 0.000s
20:01:14.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:14.445003 16375 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-0/data,/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-0/wal
20:01:14.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cd0cbe577a5e4bce9d77779ab1e537e4"
20:01:14.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:01:14 on dist-test-slave-1555"
20:01:14.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:14.445336 16375 fs_report.cc:389] FS layout report
20:01:14.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:01:14.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-0/wal
20:01:14.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-0/wal
20:01:14.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-0/data/data
20:01:14.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:01:14.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:01:14.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:01:14.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:01:14.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:01:14.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:01:14.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:01:14.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:01:14.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:01:14.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:01:14.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:01:14.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:14.533593 16375 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:01:14.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:14.543198 16375 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:01:14.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:14.543726 16375 kserver.cc:163] Server-wide thread pool size limit: 3276
20:01:14.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:14.546653 16375 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:01:14.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:14.551245 16375 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:01:14.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:14.551484 16375 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:01:14.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:14.551832 16375 ts_tablet_manager.cc:616] Registered 0 tablets
20:01:14.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:14.552027 16375 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:01:14.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:14.751605 16375 rpc_server.cc:307] RPC server started. Bound to: 127.15.168.1:36977
20:01:14.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:14.751713 16508 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.168.1:36977 every 8 connection(s)
20:01:14.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:14.754392 16375 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-0/data/info.pb
20:01:14.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:14.760113 16032 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_muOGM/build/tsan/bin/kudu as pid 16375
20:01:14.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:14.760735 16032 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-0/wal/instance
20:01:14.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:14 dist-test-slave-1555 krb5kdc[16044](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539274, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.1@KRBTEST.COM for kudu/127.15.168.60@KRBTEST.COM
20:01:14.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:14 dist-test-slave-1555 krb5kdc[16044](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539274, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.1@KRBTEST.COM for kudu/127.15.168.61@KRBTEST.COM
20:01:14.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.15.168.2@KRBTEST.COM; defaulting to no policy
20:01:14.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:01:14.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.15.168.2@KRBTEST.COM" created.
20:01:14.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:14 dist-test-slave-1555 krb5kdc[16044](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539274, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.1@KRBTEST.COM for kudu/127.15.168.62@KRBTEST.COM
20:01:14.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:14.823225 16509 heartbeater.cc:344] Connected to a master server at 127.15.168.60:45671
20:01:14.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:14.823731 16509 heartbeater.cc:461] Registering TS with master...
20:01:14.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:14.825291 16509 heartbeater.cc:507] Master 127.15.168.60:45671 requested a full tablet report, sending...
20:01:14.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:14.830259 16267 ts_manager.cc:194] Registered new tserver with Master: cd0cbe577a5e4bce9d77779ab1e537e4 (127.15.168.1:36977)
20:01:14.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:14.832948 16267 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.15.168.1@KRBTEST.COM'} at 127.15.168.1:33283
20:01:14.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:14.836652 16510 heartbeater.cc:344] Connected to a master server at 127.15.168.61:32863
20:01:14.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:14.837006 16510 heartbeater.cc:461] Registering TS with master...
20:01:14.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:14.837729 16510 heartbeater.cc:507] Master 127.15.168.61:32863 requested a full tablet report, sending...
20:01:14.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:14.841706 16181 ts_manager.cc:194] Registered new tserver with Master: cd0cbe577a5e4bce9d77779ab1e537e4 (127.15.168.1:36977)
20:01:14.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:14.845263 16511 heartbeater.cc:344] Connected to a master server at 127.15.168.62:38531
20:01:14.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:14.845634 16511 heartbeater.cc:461] Registering TS with master...
20:01:14.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:14.846292 16511 heartbeater.cc:507] Master 127.15.168.62:38531 requested a full tablet report, sending...
20:01:14.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:14.849670 16096 ts_manager.cc:194] Registered new tserver with Master: cd0cbe577a5e4bce9d77779ab1e537e4 (127.15.168.1:36977)
20:01:14.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:01:14.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.15.168.2 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab.
20:01:14.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.15.168.2 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab.
20:01:14.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.15.168.2@KRBTEST.COM; defaulting to no policy
20:01:14.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:01:14.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.15.168.2@KRBTEST.COM" created.
20:01:14.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:01:14.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.15.168.2 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab.
20:01:14.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.15.168.2 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab.
20:01:14.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:14.905795 16032 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_muOGM/build/tsan/bin/kudu
20:01:14.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_muOGM/build/tsan/bin/kudu
20:01:14.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-1/wal
20:01:14.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-1/data
20:01:14.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:01:14.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:01:14.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:01:14.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:14.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:14.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:14.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-1/logs
20:01:14.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-1/data/info.pb
20:01:14.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:14.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:01:14.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:01:14.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:01:14.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:01:14.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:14.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:14.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:14.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:01:14.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:01:14.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.168.2:0
20:01:14.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.15.168.2
20:01:14.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.168.2
20:01:14.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:01:14.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.15.168.62:38531,127.15.168.61:32863,127.15.168.60:45671
20:01:14.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab
20:01:14.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.168.2
20:01:14.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:14.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:01:14.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:01:14.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:01:14.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.168.20:32967
20:01:14.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:14.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:14.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:14.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:01:15.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:15.333516 16523 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:01:15.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:15.334088 16523 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:01:15.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:15.334762 16523 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:01:15.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:15.369943 16523 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:01:15.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:15.371034 16523 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.15.168.2
20:01:15.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:15.411309 16523 tablet_server_runner.cc:78] Tablet server non-default flags:
20:01:15.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-1/logs
20:01:15.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:01:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.168.20:32967
20:01:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-1/data
20:01:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-1/wal
20:01:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab
20:01:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.168.2
20:01:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.168.2:0
20:01:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:01:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:01:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:01:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:15.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:15.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-1/data/info.pb
20:01:15.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.168.2
20:01:15.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:01:15.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.15.168.62:38531,127.15.168.61:32863,127.15.168.60:45671
20:01:15.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:01:15.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:15.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:01:15.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:01:15.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:15.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.15.168.2
20:01:15.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:01:15.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:01:15.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:01:15.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3b18f0139b1d21a072b7353269301fba16dca54e
20:01:15.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:01:15.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Sep 2026 19:43:26 UTC on e7f111948823
20:01:15.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14030
20:01:15.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:01:15.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:15.412992 16523 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:01:15.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:15.414393 16523 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:01:15.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:15.427155 16530 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:15.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:15.837534 16509 heartbeater.cc:499] Master 127.15.168.60:45671 was elected leader, sending a full tablet report...
20:01:16.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:16 dist-test-slave-1555 krb5kdc[16044](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539276, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:16.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:16.706526 16160 init.cc:295] Successfully reacquired a new kerberos TGT
20:01:17.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:16.832017 16529 debug-util.cc:398] Leaking SignalData structure 0x7b08000372a0 after lost signal to thread 16523
20:01:17.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:17.153501 16523 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.726s user 0.577s sys 1.062s
20:01:17.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:17.153882 16523 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.726s user 0.577s sys 1.062s
20:01:17.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:15.428992 16531 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:17.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:17.158562 16533 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:17.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:17.160629 16532 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1731 milliseconds
20:01:17.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:17.161136 16523 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:01:17.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:17.162524 16523 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:01:17.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:17.169829 16523 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:01:17.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:17.171700 16523 hybrid_clock.cc:648] HybridClock initialized: now 1790539277171636 us; error 63 us; skew 500 ppm
20:01:17.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:17.172914 16523 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:01:17.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:17 dist-test-slave-1555 krb5kdc[16044](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539277, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.2@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:17.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:17.186085 16523 init.cc:377] Logged in from keytab as kudu/127.15.168.2@KRBTEST.COM (short username kudu)
20:01:17.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:17.206427 16523 webserver.cc:533] Webserver started at http://127.15.168.2:44333/ using document root <none> and password file <none>
20:01:17.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:17.208076 16523 fs_manager.cc:362] Metadata directory not provided
20:01:17.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:17.208398 16523 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:01:17.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:17.209033 16523 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:01:17.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:17.216365 16523 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-1/data/instance:
20:01:17.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "48e20487653845ea968ae05270a44097"
20:01:17.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:01:17 on dist-test-slave-1555"
20:01:17.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:17.218173 16523 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-1/wal/instance:
20:01:17.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "48e20487653845ea968ae05270a44097"
20:01:17.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:01:17 on dist-test-slave-1555"
20:01:17.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:17.227118 16523 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.005s sys 0.005s
20:01:17.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:17.233857 16544 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:01:17.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:17.235038 16523 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
20:01:17.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:17.235332 16523 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-1/data,/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-1/wal
20:01:17.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "48e20487653845ea968ae05270a44097"
20:01:17.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:01:17 on dist-test-slave-1555"
20:01:17.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:17.235626 16523 fs_report.cc:389] FS layout report
20:01:17.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:01:17.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-1/wal
20:01:17.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-1/wal
20:01:17.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-1/data/data
20:01:17.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:01:17.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:01:17.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:01:17.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:01:17.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:01:17.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:01:17.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:01:17.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:01:17.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:01:17.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:01:17.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:01:17.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:17.290616 16523 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:01:17.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:17.296667 16523 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:01:17.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:17.297077 16523 kserver.cc:163] Server-wide thread pool size limit: 3276
20:01:17.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:17.300006 16523 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:01:17.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:17.305245 16523 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:01:17.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:17.305449 16523 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:01:17.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:17.305636 16523 ts_tablet_manager.cc:616] Registered 0 tablets
20:01:17.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:17.305771 16523 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:01:17.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:17.471856 16523 rpc_server.cc:307] RPC server started. Bound to: 127.15.168.2:39021
20:01:17.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:17.472230 16657 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.168.2:39021 every 8 connection(s)
20:01:17.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:17.475288 16523 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-1/data/info.pb
20:01:17.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:17.484265 16032 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_muOGM/build/tsan/bin/kudu as pid 16523
20:01:17.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:17.484916 16032 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-1/wal/instance
20:01:17.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.15.168.3@KRBTEST.COM; defaulting to no policy
20:01:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:01:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.15.168.3@KRBTEST.COM" created.
20:01:17.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:17 dist-test-slave-1555 krb5kdc[16044](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539277, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.2@KRBTEST.COM for kudu/127.15.168.60@KRBTEST.COM
20:01:17.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:17 dist-test-slave-1555 krb5kdc[16044](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539277, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.2@KRBTEST.COM for kudu/127.15.168.61@KRBTEST.COM
20:01:17.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:17 dist-test-slave-1555 krb5kdc[16044](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539277, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.2@KRBTEST.COM for kudu/127.15.168.62@KRBTEST.COM
20:01:17.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:17.564684 16658 heartbeater.cc:344] Connected to a master server at 127.15.168.60:45671
20:01:17.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:17.565258 16658 heartbeater.cc:461] Registering TS with master...
20:01:17.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:17.566637 16658 heartbeater.cc:507] Master 127.15.168.60:45671 requested a full tablet report, sending...
20:01:17.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:17.570075 16267 ts_manager.cc:194] Registered new tserver with Master: 48e20487653845ea968ae05270a44097 (127.15.168.2:39021)
20:01:17.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:17.574057 16659 heartbeater.cc:344] Connected to a master server at 127.15.168.61:32863
20:01:17.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:17.574420 16659 heartbeater.cc:461] Registering TS with master...
20:01:17.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:17.574901 16267 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.15.168.2@KRBTEST.COM'} at 127.15.168.2:36141
20:01:17.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:17.575673 16660 heartbeater.cc:344] Connected to a master server at 127.15.168.62:38531
20:01:17.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:17.575790 16659 heartbeater.cc:507] Master 127.15.168.61:32863 requested a full tablet report, sending...
20:01:17.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:17.576037 16660 heartbeater.cc:461] Registering TS with master...
20:01:17.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:01:17.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.15.168.3 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab.
20:01:17.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.15.168.3 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab.
20:01:17.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:17.577009 16660 heartbeater.cc:507] Master 127.15.168.62:38531 requested a full tablet report, sending...
20:01:17.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:17.578831 16181 ts_manager.cc:194] Registered new tserver with Master: 48e20487653845ea968ae05270a44097 (127.15.168.2:39021)
20:01:17.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:17.585551 16096 ts_manager.cc:194] Registered new tserver with Master: 48e20487653845ea968ae05270a44097 (127.15.168.2:39021)
20:01:17.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.15.168.3@KRBTEST.COM; defaulting to no policy
20:01:17.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:01:17.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.15.168.3@KRBTEST.COM" created.
20:01:17.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:01:17.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.15.168.3 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab.
20:01:17.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.15.168.3 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab.
20:01:17.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:17.624285 16032 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_muOGM/build/tsan/bin/kudu
20:01:17.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_muOGM/build/tsan/bin/kudu
20:01:17.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-2/wal
20:01:17.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-2/data
20:01:17.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:01:17.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:01:17.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:01:17.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:17.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:17.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:17.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-2/logs
20:01:17.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-2/data/info.pb
20:01:17.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:17.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:01:17.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:01:17.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:01:17.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:01:17.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:17.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:17.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:17.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:01:17.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:01:17.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.168.3:0
20:01:17.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.15.168.3
20:01:17.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.168.3
20:01:17.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:01:17.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.15.168.62:38531,127.15.168.61:32863,127.15.168.60:45671
20:01:17.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab
20:01:17.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.168.3
20:01:17.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:17.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:01:17.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:01:17.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:01:17.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.168.20:32967
20:01:17.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:17.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:17.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:17.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:01:18.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:18.005191 16672 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:01:18.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:18.005667 16672 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:01:18.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:18.006167 16672 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:01:18.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:18.022959 15536 debug-util.cc:398] Leaking SignalData structure 0x7b0800089920 after lost signal to thread 15467
20:01:18.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:18.023725 15536 debug-util.cc:398] Leaking SignalData structure 0x7b08000b47c0 after lost signal to thread 15539
20:01:18.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:18.043257 16672 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:01:18.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:18.044425 16672 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.15.168.3
20:01:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:18.089025 16672 tablet_server_runner.cc:78] Tablet server non-default flags:
20:01:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-2/logs
20:01:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:01:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.168.20:32967
20:01:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-2/data
20:01:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-2/wal
20:01:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab
20:01:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.168.3
20:01:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.168.3:0
20:01:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:01:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:01:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:01:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-2/data/info.pb
20:01:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.168.3
20:01:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:01:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.15.168.62:38531,127.15.168.61:32863,127.15.168.60:45671
20:01:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:01:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:01:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:01:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.15.168.3
20:01:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:01:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:01:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:01:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3b18f0139b1d21a072b7353269301fba16dca54e
20:01:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:01:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Sep 2026 19:43:26 UTC on e7f111948823
20:01:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14030
20:01:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:01:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:18.090701 16672 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:01:18.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:18.091780 16672 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:01:18.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:18.106441 16680 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:18.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:18.207628 16015 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad8a0 after lost signal to thread 15890
20:01:18.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:18.209585 16015 debug-util.cc:398] Leaking SignalData structure 0x7b08000ba320 after lost signal to thread 16018
20:01:18.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:18.582432 16658 heartbeater.cc:499] Master 127.15.168.60:45671 was elected leader, sending a full tablet report...
20:01:18.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:18 dist-test-slave-1555 krb5kdc[16044](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539278, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:18.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:18.837057 16246 init.cc:295] Successfully reacquired a new kerberos TGT
20:01:20.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:19.507601 16679 debug-util.cc:398] Leaking SignalData structure 0x7b08000372a0 after lost signal to thread 16672
20:01:20.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:20.025789 16679 kernel_stack_watchdog.cc:198] Thread 16672 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 398ms:
20:01:20.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:01:20.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:01:20.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:01:20.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:01:20.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:20.028586 16672 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.920s user 0.635s sys 0.980s
20:01:20.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:20.029621 16672 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.922s user 0.635s sys 0.980s
20:01:20.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:18.111356 16681 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:20.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:20.039535 16672 server_base.cc:1061] running on GCE node
20:01:20.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:20.042243 16687 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:20.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:20.043638 16672 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:01:20.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:20.052073 16672 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:01:20.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:20.053622 16672 hybrid_clock.cc:648] HybridClock initialized: now 1790539280053582 us; error 43 us; skew 500 ppm
20:01:20.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:20.054633 16672 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:01:20.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:20 dist-test-slave-1555 krb5kdc[16044](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539280, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.3@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:20.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:20.061793 16672 init.cc:377] Logged in from keytab as kudu/127.15.168.3@KRBTEST.COM (short username kudu)
20:01:20.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:20.068440 16672 webserver.cc:533] Webserver started at http://127.15.168.3:46233/ using document root <none> and password file <none>
20:01:20.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:20.069383 16672 fs_manager.cc:362] Metadata directory not provided
20:01:20.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:20.069659 16672 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:01:20.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:20.070240 16672 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:01:20.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:20.075326 16672 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-2/data/instance:
20:01:20.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "83fbdd41d37846738f806b86554e3f41"
20:01:20.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:01:20 on dist-test-slave-1555"
20:01:20.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:20.076683 16672 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-2/wal/instance:
20:01:20.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "83fbdd41d37846738f806b86554e3f41"
20:01:20.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:01:20 on dist-test-slave-1555"
20:01:20.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:20.086473 16672 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.007s sys 0.000s
20:01:20.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:20.093055 16693 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:01:20.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:20.094319 16672 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
20:01:20.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:20.094681 16672 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-2/data,/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-2/wal
20:01:20.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "83fbdd41d37846738f806b86554e3f41"
20:01:20.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:01:20 on dist-test-slave-1555"
20:01:20.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:20.095037 16672 fs_report.cc:389] FS layout report
20:01:20.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:01:20.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-2/wal
20:01:20.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-2/wal
20:01:20.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-2/data/data
20:01:20.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:01:20.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:01:20.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:01:20.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:01:20.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:01:20.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:01:20.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:01:20.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:01:20.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:01:20.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:01:20.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:01:20.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:20.179774 16672 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:01:20.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:20.193400 16672 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:01:20.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:20.193994 16672 kserver.cc:163] Server-wide thread pool size limit: 3276
20:01:20.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:20.198009 16672 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:01:20.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:20.205744 16672 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:01:20.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:20.206012 16672 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:01:20.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:20.206391 16672 ts_tablet_manager.cc:616] Registered 0 tablets
20:01:20.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:20.206673 16672 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:01:20.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:20.672947 16672 rpc_server.cc:307] RPC server started. Bound to: 127.15.168.3:35707
20:01:20.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:20.673143 16805 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.168.3:35707 every 8 connection(s)
20:01:20.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:20.676360 16672 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-2/data/info.pb
20:01:20.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:20.678519 16032 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_muOGM/build/tsan/bin/kudu as pid 16672
20:01:20.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:20.679471 16032 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/ts-2/wal/instance
20:01:20.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:20 dist-test-slave-1555 krb5kdc[16044](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539280, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.3@KRBTEST.COM for kudu/127.15.168.61@KRBTEST.COM
20:01:20.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:20 dist-test-slave-1555 krb5kdc[16044](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539280, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.3@KRBTEST.COM for kudu/127.15.168.60@KRBTEST.COM
20:01:20.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:20 dist-test-slave-1555 krb5kdc[16044](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539280, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.3@KRBTEST.COM for kudu/127.15.168.62@KRBTEST.COM
20:01:20.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:20.906840 16812 heartbeater.cc:344] Connected to a master server at 127.15.168.62:38531
20:01:20.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:20.907400 16812 heartbeater.cc:461] Registering TS with master...
20:01:20.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:20.909034 16812 heartbeater.cc:507] Master 127.15.168.62:38531 requested a full tablet report, sending...
20:01:20.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:20.912443 16096 ts_manager.cc:194] Registered new tserver with Master: 83fbdd41d37846738f806b86554e3f41 (127.15.168.3:35707)
20:01:20.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:20.918159 16807 heartbeater.cc:344] Connected to a master server at 127.15.168.61:32863
20:01:20.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:20.918522 16807 heartbeater.cc:461] Registering TS with master...
20:01:20.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:20.919510 16807 heartbeater.cc:507] Master 127.15.168.61:32863 requested a full tablet report, sending...
20:01:20.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:20.920399 16806 heartbeater.cc:344] Connected to a master server at 127.15.168.60:45671
20:01:20.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:20.920727 16806 heartbeater.cc:461] Registering TS with master...
20:01:20.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:20.921787 16806 heartbeater.cc:507] Master 127.15.168.60:45671 requested a full tablet report, sending...
20:01:20.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:20.922737 16181 ts_manager.cc:194] Registered new tserver with Master: 83fbdd41d37846738f806b86554e3f41 (127.15.168.3:35707)
20:01:20.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:20.924705 16267 ts_manager.cc:194] Registered new tserver with Master: 83fbdd41d37846738f806b86554e3f41 (127.15.168.3:35707)
20:01:20.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:20.926679 16267 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.15.168.3@KRBTEST.COM'} at 127.15.168.3:45981
20:01:20.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:20.943989 16032 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
20:01:20.966 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:20.967 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
20:01:20.978 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
20:01:20.979 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
20:01:20.984 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.15.168.62"
port: 38531
}
bound_http_address {
host: "127.15.168.62"
port: 41983
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.15.168.61"
port: 32863
}
bound_http_address {
host: "127.15.168.61"
port: 35057
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.15.168.60"
port: 45671
}
bound_http_address {
host: "127.15.168.60"
port: 33645
}
}
}
20:01:20.984 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
20:01:20.985 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.15.168.1"
port: 36977
}
bound_http_address {
host: "127.15.168.1"
port: 35773
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.15.168.2"
port: 39021
}
bound_http_address {
host: "127.15.168.2"
port: 44333
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.15.168.3"
port: 35707
}
bound_http_address {
host: "127.15.168.3"
port: 46233
}
}
}
20:01:20.986 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
20:01:20.987 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
20:01:21.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:21 dist-test-slave-1555 krb5kdc[16044](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539281, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:21.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:01:21.039 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:21.043 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/krb5cc
20:01:21.057 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:01:21.059 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/krb5cc
20:01:21.060 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:01:21.097 [DEBUG - Time-limited test] (SecurityContext.java:158) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
20:01:21.100 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
20:01:21.100 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
20:01:21.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:21 dist-test-slave-1555 krb5kdc[16044](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539281, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:21.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:01:21.146 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:21.147 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/krb5cc
20:01:21.149 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:01:21.149 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.15.168.61:32863
20:01:21.151 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
20:01:21.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:21.152387 16032 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_muOGM/build/tsan/bin/kudu with pid 16140
20:01:21.192 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:21.192 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.15.168.62:38531
20:01:21.192 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
20:01:21.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:21.193425 16032 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_muOGM/build/tsan/bin/kudu with pid 16061
20:01:21.242 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:21.242 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.15.168.60:45671
20:01:21.242 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
20:01:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:21.243072 16032 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_muOGM/build/tsan/bin/kudu with pid 16227
20:01:21.309 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:21.310 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.15.168.61:32863
20:01:21.315 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
20:01:21.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:21.317044 16032 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_muOGM/build/tsan/bin/kudu
20:01:21.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_muOGM/build/tsan/bin/kudu
20:01:21.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/wal
20:01:21.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/data
20:01:21.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:01:21.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:01:21.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:01:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/logs
20:01:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/data/info.pb
20:01:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:01:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:01:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:01:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:01:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:01:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:01:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.168.61:32863
20:01:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.168.61
20:01:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35057
20:01:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab
20:01:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.168.61
20:01:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:01:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:01:21.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:01:21.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.168.20:32967
20:01:21.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:21.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:21.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:21.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:21.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.168.62:38531,127.15.168.61:32863,127.15.168.60:45671
20:01:21.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:21.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41913/jwks.json
20:01:21.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:01:21.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:01:21.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:21.611145 16658 heartbeater.cc:646] Failed to heartbeat to 127.15.168.60:45671 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.15.168.60:45671: connect: Connection refused (error 111)
20:01:21.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:21.703128 16820 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:01:21.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:21.703661 16820 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:01:21.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:21.704348 16820 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:01:21.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:21.749792 16820 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:01:21.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:21.750124 16820 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:01:21.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:21.750404 16820 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:01:21.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:21.750710 16820 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:01:21.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:21.751062 16820 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:01:21.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:21.751296 16820 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:01:21.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:21.751452 16820 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:41913/jwks.json
20:01:21.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:21.790850 16820 master_runner.cc:387] Master server non-default flags:
20:01:21.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/logs
20:01:21.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:21.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:01:21.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:21.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.168.20:32967
20:01:21.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:21.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:21.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/data
20:01:21.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/wal
20:01:21.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:01:21.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:21.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.168.62:38531,127.15.168.61:32863,127.15.168.60:45671
20:01:21.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab
20:01:21.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.168.61
20:01:21.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:21.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:21.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:21.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.168.61:32863
20:01:21.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:21.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:01:21.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:21.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41913/jwks.json
20:01:21.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:01:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:01:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/data/info.pb
20:01:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.168.61
20:01:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35057
20:01:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:01:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:01:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:01:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:01:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:01:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:01:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3b18f0139b1d21a072b7353269301fba16dca54e
20:01:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:01:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Sep 2026 19:43:26 UTC on e7f111948823
20:01:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14030
20:01:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:01:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:21.792785 16820 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:01:21.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:21.794358 16820 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:01:21.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:21.806126 16830 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:21.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:21.930392 16806 heartbeater.cc:499] Master 127.15.168.60:45671 was elected leader, sending a full tablet report...
20:01:21.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:21.937675 16806 heartbeater.cc:646] Failed to heartbeat to 127.15.168.60:45671 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.15.168.60:45671: connect: Connection refused (error 111)
20:01:21.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:21.941565 16509 heartbeater.cc:646] Failed to heartbeat to 127.15.168.60:45671 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.15.168.60:45671: connect: Connection refused (error 111)
20:01:23.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:23.210886 16829 debug-util.cc:398] Leaking SignalData structure 0x7b0800037cc0 after lost signal to thread 16820
20:01:23.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:23.623250 16829 kernel_stack_watchdog.cc:198] Thread 16820 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 401ms:
20:01:23.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:01:23.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:01:23.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:01:23.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:01:23.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:21.807061 16831 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:23.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:23.624590 16820 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.818s user 0.639s sys 1.103s
20:01:23.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:23.624945 16820 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.819s user 0.639s sys 1.103s
20:01:23.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:23.626539 16833 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:23.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:23.628252 16832 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1816 milliseconds
20:01:23.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:23.628284 16820 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:01:23.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:23.629595 16820 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:01:23.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:23.632037 16820 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:01:23.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:23.633481 16820 hybrid_clock.cc:648] HybridClock initialized: now 1790539283633433 us; error 41 us; skew 500 ppm
20:01:23.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:23.634473 16820 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:01:23.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:23 dist-test-slave-1555 krb5kdc[16044](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539283, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:23.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:23.641706 16820 init.cc:377] Logged in from keytab as kudu/127.15.168.61@KRBTEST.COM (short username kudu)
20:01:23.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:23.646966 16820 webserver.cc:533] Webserver started at http://127.15.168.61:35057/ using document root <none> and password file <none>
20:01:23.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:23.647953 16820 fs_manager.cc:362] Metadata directory not provided
20:01:23.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:23.648265 16820 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:01:23.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:23.656577 16820 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.007s sys 0.000s
20:01:23.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:23.661113 16856 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:01:23.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:23.662572 16820 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
20:01:23.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:23.662956 16820 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/data,/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/wal
20:01:23.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4b00615a9e9244d2a54e01564420e235"
20:01:23.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:01:06 on dist-test-slave-1555"
20:01:23.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:23.664968 16820 fs_report.cc:389] FS layout report
20:01:23.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:01:23.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/wal
20:01:23.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/wal
20:01:23.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/data/data
20:01:23.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:01:23.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:01:23.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:01:23.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:01:23.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:01:23.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:01:23.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:01:23.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:01:23.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:01:23.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:01:23.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:01:23.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:23.720624 16820 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:01:23.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:23.726608 16820 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:01:23.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:23.727133 16820 kserver.cc:163] Server-wide thread pool size limit: 3276
20:01:23.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:23.806591 16820 rpc_server.cc:307] RPC server started. Bound to: 127.15.168.61:32863
20:01:23.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:23.806696 16907 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.168.61:32863 every 8 connection(s)
20:01:23.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:23.809579 16820 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/data/info.pb
20:01:23.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:23.816015 16032 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_muOGM/build/tsan/bin/kudu as pid 16820
20:01:23.816 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:23.817 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.15.168.62:38531
20:01:23.817 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
20:01:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:23.819151 16032 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_muOGM/build/tsan/bin/kudu
20:01:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_muOGM/build/tsan/bin/kudu
20:01:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/wal
20:01:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/data
20:01:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:01:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:01:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:01:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/logs
20:01:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/data/info.pb
20:01:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:01:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:01:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:01:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:01:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:01:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:01:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.168.62:38531
20:01:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.168.62
20:01:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41983
20:01:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab
20:01:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.168.62
20:01:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:01:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:01:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:01:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.168.20:32967
20:01:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.168.62:38531,127.15.168.61:32863,127.15.168.60:45671
20:01:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41913/jwks.json
20:01:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:01:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:01:23.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:23.821823 16908 sys_catalog.cc:263] Verifying existing consensus state
20:01:23.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:23.827972 16908 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235: Bootstrap starting.
20:01:23.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:23.867936 16908 log.cc:826] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235: Log is configured to *not* fsync() on all Append() calls
20:01:23.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:23.883445 16908 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:01:23.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:23.884452 16908 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235: Bootstrap complete.
20:01:23.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:23.920437 16908 raft_consensus.cc:359] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } }
20:01:23.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:23.922660 16908 raft_consensus.cc:740] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4b00615a9e9244d2a54e01564420e235, State: Initialized, Role: FOLLOWER
20:01:23.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:23.923619 16908 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } }
20:01:23.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:23.926262 16912 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } } }
20:01:23.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:23.927256 16912 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [sys.catalog]: This master's current role is: FOLLOWER
20:01:23.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:23.927404 16908 sys_catalog.cc:565] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [sys.catalog]: configured and running, proceeding with master startup.
20:01:23.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:23.964903 16923 catalog_manager.cc:1308] Loaded cluster ID: d6c21a38473c4d4189831994cb48f2ce
20:01:23.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:23.965231 16923 catalog_manager.cc:1601] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235: loading cluster ID for follower catalog manager: success
20:01:23.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:23.973284 16923 catalog_manager.cc:1623] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235: acquiring CA information for follower catalog manager: success
20:01:23.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:23.979148 16923 catalog_manager.cc:1651] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:01:23.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:23.954839 16820 master_runner.cc:187] Error getting master registration for 127.15.168.62:38531: Network error: Client connection negotiation failed: client connection to 127.15.168.62:38531: connect: Connection refused (error 111),
20:01:23.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:23.990152 16820 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:01:24.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:24.009094 16807 heartbeater.cc:344] Connected to a master server at 127.15.168.61:32863
20:01:24.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:24.015396 16873 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "83fbdd41d37846738f806b86554e3f41" instance_seqno: 1790539280611250) as {username='kudu', principal='kudu/127.15.168.3@KRBTEST.COM'} at 127.15.168.3:42223; Asking this server to re-register.
20:01:24.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:24.016979 16807 heartbeater.cc:461] Registering TS with master...
20:01:24.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:24.019708 16873 ts_manager.cc:194] Registered new tserver with Master: 83fbdd41d37846738f806b86554e3f41 (127.15.168.3:35707)
20:01:24.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:24.055382 16510 heartbeater.cc:344] Connected to a master server at 127.15.168.61:32863
20:01:24.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:24.058259 16873 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "cd0cbe577a5e4bce9d77779ab1e537e4" instance_seqno: 1790539274711308) as {username='kudu', principal='kudu/127.15.168.1@KRBTEST.COM'} at 127.15.168.1:58201; Asking this server to re-register.
20:01:24.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:24.059664 16510 heartbeater.cc:461] Registering TS with master...
20:01:24.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:24.063153 16873 ts_manager.cc:194] Registered new tserver with Master: cd0cbe577a5e4bce9d77779ab1e537e4 (127.15.168.1:36977)
20:01:24.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:24.186489 16910 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:01:24.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:24.187186 16910 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:01:24.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:24.187966 16910 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:01:24.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:24.222698 16910 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:01:24.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:24.223001 16910 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:01:24.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:24.223433 16910 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:01:24.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:24.223778 16910 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:01:24.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:24.224138 16910 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:01:24.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:24.224475 16910 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:01:24.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:24.224725 16910 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:41913/jwks.json
20:01:24.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:24.266316 16910 master_runner.cc:387] Master server non-default flags:
20:01:24.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/logs
20:01:24.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:24.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:01:24.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:24.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.168.20:32967
20:01:24.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:24.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:24.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/data
20:01:24.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/wal
20:01:24.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:01:24.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:24.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.168.62:38531,127.15.168.61:32863,127.15.168.60:45671
20:01:24.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab
20:01:24.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.168.62
20:01:24.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:24.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:24.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:24.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.168.62:38531
20:01:24.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:24.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:01:24.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:24.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41913/jwks.json
20:01:24.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:24.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:01:24.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:01:24.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:24.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:24.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/data/info.pb
20:01:24.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.168.62
20:01:24.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41983
20:01:24.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:01:24.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:24.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:01:24.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:01:24.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:24.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:01:24.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:01:24.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:01:24.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3b18f0139b1d21a072b7353269301fba16dca54e
20:01:24.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:01:24.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Sep 2026 19:43:26 UTC on e7f111948823
20:01:24.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14030
20:01:24.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:01:24.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:24.267953 16910 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:01:24.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:24.269469 16910 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:01:24.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:24.281395 16935 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:24.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:24 dist-test-slave-1555 krb5kdc[16044](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539284, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.1@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:24.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:24.408701 16390 init.cc:295] Successfully reacquired a new kerberos TGT
20:01:24.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:24.666085 16659 heartbeater.cc:344] Connected to a master server at 127.15.168.61:32863
20:01:24.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:24.670400 16873 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "48e20487653845ea968ae05270a44097" instance_seqno: 1790539277425272) as {username='kudu', principal='kudu/127.15.168.2@KRBTEST.COM'} at 127.15.168.2:42169; Asking this server to re-register.
20:01:24.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:24.672308 16659 heartbeater.cc:461] Registering TS with master...
20:01:24.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:24.675580 16873 ts_manager.cc:194] Registered new tserver with Master: 48e20487653845ea968ae05270a44097 (127.15.168.2:39021)
20:01:25.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:25.508466 16951 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:01:25.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:25.510198 16951 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } }
20:01:25.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:25.530164 16951 leader_election.cc:290] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 95585e2dbd5a475f9466ce816f5fffff (127.15.168.62:38531), 33ab2c6f07d14efa88f25702047f032c (127.15.168.60:45671)
20:01:25.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:25.569288 16859 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.168.60:45671: connect: Connection refused (error 111)
20:01:25.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:25.605726 16859 leader_election.cc:336] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 33ab2c6f07d14efa88f25702047f032c (127.15.168.60:45671): Network error: Client connection negotiation failed: client connection to 127.15.168.60:45671: connect: Connection refused (error 111)
20:01:25.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:25.610915 16860 leader_election.cc:336] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 95585e2dbd5a475f9466ce816f5fffff (127.15.168.62:38531): Network error: Client connection negotiation failed: client connection to 127.15.168.62:38531: connect: Connection refused (error 111)
20:01:25.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:25.611877 16860 leader_election.cc:304] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4b00615a9e9244d2a54e01564420e235; no voters: 33ab2c6f07d14efa88f25702047f032c, 95585e2dbd5a475f9466ce816f5fffff
20:01:25.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:25.615022 16951 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
20:01:25.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:24.283586 16936 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:25.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:24.306743 16938 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:25.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:25.917755 16937 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1633 milliseconds
20:01:25.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:25.917775 16910 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:01:25.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:25.922897 16910 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:01:25.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:25.926256 16910 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:01:25.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:25.927737 16910 hybrid_clock.cc:648] HybridClock initialized: now 1790539285927666 us; error 83 us; skew 500 ppm
20:01:25.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:25.928902 16910 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:01:25.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:25 dist-test-slave-1555 krb5kdc[16044](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539285, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:25.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:25.936331 16910 init.cc:377] Logged in from keytab as kudu/127.15.168.62@KRBTEST.COM (short username kudu)
20:01:25.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:25.942581 16910 webserver.cc:533] Webserver started at http://127.15.168.62:41983/ using document root <none> and password file <none>
20:01:25.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:25.943609 16910 fs_manager.cc:362] Metadata directory not provided
20:01:25.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:25.943969 16910 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:01:25.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:25.953292 16910 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.004s sys 0.004s
20:01:25.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:25.958715 16962 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:01:25.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:25.960027 16910 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.005s sys 0.000s
20:01:25.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:25.960378 16910 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/data,/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/wal
20:01:25.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "95585e2dbd5a475f9466ce816f5fffff"
20:01:25.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:01:04 on dist-test-slave-1555"
20:01:25.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:25.962303 16910 fs_report.cc:389] FS layout report
20:01:25.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:01:25.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/wal
20:01:25.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/wal
20:01:25.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/data/data
20:01:25.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:01:25.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:01:25.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:01:25.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:01:25.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:01:25.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:01:25.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:01:25.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:01:25.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:01:25.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:01:25.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:01:26.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:26.022919 16910 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:01:26.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:26.031689 16910 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:01:26.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:26.032316 16910 kserver.cc:163] Server-wide thread pool size limit: 3276
20:01:26.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:26.127575 16910 rpc_server.cc:307] RPC server started. Bound to: 127.15.168.62:38531
20:01:26.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:26.127617 17015 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.168.62:38531 every 8 connection(s)
20:01:26.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:26.131594 16910 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/data/info.pb
20:01:26.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:26.132673 16032 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_muOGM/build/tsan/bin/kudu as pid 16910
20:01:26.134 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:26.134 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.15.168.60:45671
20:01:26.134 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
20:01:26.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:26.136505 16032 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_muOGM/build/tsan/bin/kudu
20:01:26.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_muOGM/build/tsan/bin/kudu
20:01:26.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/wal
20:01:26.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/data
20:01:26.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:01:26.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:01:26.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:01:26.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:26.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:26.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:26.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/logs
20:01:26.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/data/info.pb
20:01:26.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:26.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:01:26.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:01:26.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:01:26.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:01:26.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:26.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:26.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:26.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:01:26.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:01:26.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:26.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:26.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.168.60:45671
20:01:26.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.168.60
20:01:26.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33645
20:01:26.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab
20:01:26.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.168.60
20:01:26.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:26.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:01:26.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:01:26.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:01:26.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.168.20:32967
20:01:26.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:26.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:26.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:26.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:26.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.168.62:38531,127.15.168.61:32863,127.15.168.60:45671
20:01:26.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:26.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41913/jwks.json
20:01:26.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:01:26.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:01:26.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:26.149117 17016 sys_catalog.cc:263] Verifying existing consensus state
20:01:26.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:26.158847 17016 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff: Bootstrap starting.
20:01:26.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:26.217301 17016 log.cc:826] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff: Log is configured to *not* fsync() on all Append() calls
20:01:26.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:26.240264 17016 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:01:26.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:26.241405 17016 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff: Bootstrap complete.
20:01:26.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:26.278887 17016 raft_consensus.cc:359] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } }
20:01:26.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:26.283039 17016 raft_consensus.cc:740] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 95585e2dbd5a475f9466ce816f5fffff, State: Initialized, Role: FOLLOWER
20:01:26.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:26.284408 17016 consensus_queue.cc:260] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } }
20:01:26.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:26.306546 17020 sys_catalog.cc:455] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } } }
20:01:26.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:26.310133 17020 sys_catalog.cc:458] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [sys.catalog]: This master's current role is: FOLLOWER
20:01:26.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:26.308694 17016 sys_catalog.cc:565] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [sys.catalog]: configured and running, proceeding with master startup.
20:01:26.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:26.408908 17031 catalog_manager.cc:1308] Loaded cluster ID: d6c21a38473c4d4189831994cb48f2ce
20:01:26.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:26.409202 17031 catalog_manager.cc:1601] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff: loading cluster ID for follower catalog manager: success
20:01:26.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:26.418897 17031 catalog_manager.cc:1623] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff: acquiring CA information for follower catalog manager: success
20:01:26.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:26.435042 17031 catalog_manager.cc:1651] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:01:26.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:26.541065 16910 master_runner.cc:187] Error getting master registration for 127.15.168.60:45671: Network error: Client connection negotiation failed: client connection to 127.15.168.60:45671: connect: Connection refused (error 111),
20:01:26.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:26.572300 16910 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:01:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:26.655681 17018 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:01:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:26.656433 17018 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:01:26.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:26.657205 17018 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:01:26.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:26.717475 16660 heartbeater.cc:344] Connected to a master server at 127.15.168.62:38531
20:01:26.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:26.723702 16981 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "48e20487653845ea968ae05270a44097" instance_seqno: 1790539277425272) as {username='kudu', principal='kudu/127.15.168.2@KRBTEST.COM'} at 127.15.168.2:49563; Asking this server to re-register.
20:01:26.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:26.725822 16660 heartbeater.cc:461] Registering TS with master...
20:01:26.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:26.729732 16980 ts_manager.cc:194] Registered new tserver with Master: 48e20487653845ea968ae05270a44097 (127.15.168.2:39021)
20:01:26.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:26.749253 17018 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:01:26.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:26.749596 17018 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:01:26.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:26.750090 17018 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:01:26.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:26.750444 17018 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:01:26.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:26.750789 17018 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:01:26.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:26.751142 17018 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:01:26.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:26.751398 17018 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:41913/jwks.json
20:01:26.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:26.823000 17018 master_runner.cc:387] Master server non-default flags:
20:01:26.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/logs
20:01:26.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:26.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:01:26.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:26.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.168.20:32967
20:01:26.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:26.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:26.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/data
20:01:26.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/wal
20:01:26.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:01:26.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:26.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.168.62:38531,127.15.168.61:32863,127.15.168.60:45671
20:01:26.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab
20:01:26.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.168.60
20:01:26.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:26.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:26.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:26.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.168.60:45671
20:01:26.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:26.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:01:26.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:26.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41913/jwks.json
20:01:26.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:26.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:01:26.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:01:26.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:26.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:26.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/data/info.pb
20:01:26.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.168.60
20:01:26.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33645
20:01:26.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:01:26.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:26.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:01:26.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:01:26.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:26.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:01:26.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:01:26.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:01:26.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3b18f0139b1d21a072b7353269301fba16dca54e
20:01:26.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:01:26.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Sep 2026 19:43:26 UTC on e7f111948823
20:01:26.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14030
20:01:26.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:01:26.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:26.824744 17018 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:01:26.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:26.826094 17018 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:01:26.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:26.860383 17044 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:27.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:27.099789 16812 heartbeater.cc:344] Connected to a master server at 127.15.168.62:38531
20:01:27.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:27.103852 16980 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "83fbdd41d37846738f806b86554e3f41" instance_seqno: 1790539280611250) as {username='kudu', principal='kudu/127.15.168.3@KRBTEST.COM'} at 127.15.168.3:45873; Asking this server to re-register.
20:01:27.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:27.106323 16812 heartbeater.cc:461] Registering TS with master...
20:01:27.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:27.110854 16980 ts_manager.cc:194] Registered new tserver with Master: 83fbdd41d37846738f806b86554e3f41 (127.15.168.3:35707)
20:01:27.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:27.141788 16511 heartbeater.cc:344] Connected to a master server at 127.15.168.62:38531
20:01:27.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:27.145365 16980 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "cd0cbe577a5e4bce9d77779ab1e537e4" instance_seqno: 1790539274711308) as {username='kudu', principal='kudu/127.15.168.1@KRBTEST.COM'} at 127.15.168.1:53273; Asking this server to re-register.
20:01:27.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:27.147317 16511 heartbeater.cc:461] Registering TS with master...
20:01:27.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:27.150600 16980 ts_manager.cc:194] Registered new tserver with Master: cd0cbe577a5e4bce9d77779ab1e537e4 (127.15.168.1:36977)
20:01:27.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:27 dist-test-slave-1555 krb5kdc[16044](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539287, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.2@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:27.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:27.202814 16540 init.cc:295] Successfully reacquired a new kerberos TGT
20:01:27.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:27.657910 17054 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:01:27.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:27.659188 17054 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } }
20:01:27.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:27.692040 17054 leader_election.cc:290] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 95585e2dbd5a475f9466ce816f5fffff (127.15.168.62:38531), 33ab2c6f07d14efa88f25702047f032c (127.15.168.60:45671)
20:01:27.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:27.697798 16991 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4b00615a9e9244d2a54e01564420e235" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "95585e2dbd5a475f9466ce816f5fffff" is_pre_election: true
20:01:27.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:27.698789 16991 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4b00615a9e9244d2a54e01564420e235 in term 1.
20:01:27.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:27.700591 16860 leader_election.cc:304] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4b00615a9e9244d2a54e01564420e235, 95585e2dbd5a475f9466ce816f5fffff; no voters:
20:01:27.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:27.704566 17054 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [term 1 FOLLOWER]: Leader pre-election won for term 2
20:01:27.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:27.708454 17054 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:01:27.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:27.709223 17054 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [term 1 FOLLOWER]: Advancing to term 2
20:01:27.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:27.740592 16859 leader_election.cc:336] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 33ab2c6f07d14efa88f25702047f032c (127.15.168.60:45671): Network error: Client connection negotiation failed: client connection to 127.15.168.60:45671: connect: Connection refused (error 111)
20:01:27.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:27.755973 17054 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } }
20:01:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:27.763221 16991 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4b00615a9e9244d2a54e01564420e235" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "95585e2dbd5a475f9466ce816f5fffff"
20:01:27.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:27.764675 16991 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [term 1 FOLLOWER]: Advancing to term 2
20:01:27.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:27.780453 17054 leader_election.cc:290] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [CANDIDATE]: Term 2 election: Requested vote from peers 95585e2dbd5a475f9466ce816f5fffff (127.15.168.62:38531), 33ab2c6f07d14efa88f25702047f032c (127.15.168.60:45671)
20:01:27.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:27.781315 16859 leader_election.cc:336] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 33ab2c6f07d14efa88f25702047f032c (127.15.168.60:45671): Network error: Client connection negotiation failed: client connection to 127.15.168.60:45671: connect: Connection refused (error 111)
20:01:27.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:27.790114 16991 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4b00615a9e9244d2a54e01564420e235 in term 2.
20:01:27.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:27.795831 16860 leader_election.cc:304] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4b00615a9e9244d2a54e01564420e235, 95585e2dbd5a475f9466ce816f5fffff; no voters: 33ab2c6f07d14efa88f25702047f032c
20:01:27.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:27.798151 17054 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [term 2 FOLLOWER]: Leader election won for term 2
20:01:27.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:27.807693 17054 raft_consensus.cc:697] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [term 2 LEADER]: Becoming Leader. State: Replica: 4b00615a9e9244d2a54e01564420e235, State: Running, Role: LEADER
20:01:27.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:27.809870 17054 consensus_queue.cc:237] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } }
20:01:27.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:27.837476 17060 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4b00615a9e9244d2a54e01564420e235. Latest consensus state: current_term: 2 leader_uuid: "4b00615a9e9244d2a54e01564420e235" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } } }
20:01:27.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:27.839996 17060 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [sys.catalog]: This master's current role is: LEADER
20:01:27.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:27.846287 17062 catalog_manager.cc:1524] Loading table and tablet metadata into memory...
20:01:27.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:27.865871 17062 catalog_manager.cc:1533] Initializing Kudu cluster ID...
20:01:27.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:27.878760 17062 catalog_manager.cc:1308] Loaded cluster ID: d6c21a38473c4d4189831994cb48f2ce
20:01:27.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:27.879812 17062 catalog_manager.cc:1544] Initializing Kudu internal certificate authority...
20:01:27.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:27.889541 17062 catalog_manager.cc:1553] Loading token signing keys...
20:01:27.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:27.896572 17062 catalog_manager.cc:6230] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235: Loaded TSK: 0
20:01:27.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:27.899209 17062 catalog_manager.cc:1563] Initializing in-progress tserver states...
20:01:28.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:28.235082 16991 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [term 2 FOLLOWER]: Refusing update from remote peer 4b00615a9e9244d2a54e01564420e235: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
20:01:28.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:28.237432 17060 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [LEADER]: Connected to new peer: Peer: permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
20:01:28.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:28.263733 17066 sys_catalog.cc:455] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4b00615a9e9244d2a54e01564420e235. Latest consensus state: current_term: 2 leader_uuid: "4b00615a9e9244d2a54e01564420e235" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } } }
20:01:28.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:28.269371 17066 sys_catalog.cc:458] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [sys.catalog]: This master's current role is: FOLLOWER
20:01:28.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:28.280237 17066 sys_catalog.cc:455] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "4b00615a9e9244d2a54e01564420e235" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } } }
20:01:28.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:28.280927 17066 sys_catalog.cc:458] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [sys.catalog]: This master's current role is: FOLLOWER
20:01:28.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:28.305503 17060 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "4b00615a9e9244d2a54e01564420e235" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } } }
20:01:28.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:28.306252 17060 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [sys.catalog]: This master's current role is: LEADER
20:01:28.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:28.329826 16859 consensus_peers.cc:597] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 -> Peer 33ab2c6f07d14efa88f25702047f032c (127.15.168.60:45671): Couldn't send request to peer 33ab2c6f07d14efa88f25702047f032c. Status: Network error: Client connection negotiation failed: client connection to 127.15.168.60:45671: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:01:28.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:26.863642 17045 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:28.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:28.254161 17043 debug-util.cc:398] Leaking SignalData structure 0x7b0800037cc0 after lost signal to thread 17018
20:01:28.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:28.575546 17018 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.714s user 0.461s sys 0.836s
20:01:28.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:28.576700 17018 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.715s user 0.461s sys 0.836s
20:01:28.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:28.589896 17018 server_base.cc:1061] running on GCE node
20:01:28.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:28.590808 17073 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:28.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:28.592314 17018 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:01:28.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:28.595575 17018 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:01:28.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:28.597024 17018 hybrid_clock.cc:648] HybridClock initialized: now 1790539288596879 us; error 132 us; skew 500 ppm
20:01:28.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:28.597872 17018 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:01:28.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:28 dist-test-slave-1555 krb5kdc[16044](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539288, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:28.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:28.604311 17018 init.cc:377] Logged in from keytab as kudu/127.15.168.60@KRBTEST.COM (short username kudu)
20:01:28.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:28.610332 17018 webserver.cc:533] Webserver started at http://127.15.168.60:33645/ using document root <none> and password file <none>
20:01:28.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:28.611560 17018 fs_manager.cc:362] Metadata directory not provided
20:01:28.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:28.611868 17018 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:01:28.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:28.623301 17018 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.006s sys 0.000s
20:01:28.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:28.628363 17079 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:01:28.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:28.629730 17018 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.003s
20:01:28.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:28.630021 17018 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/data,/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/wal
20:01:28.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "33ab2c6f07d14efa88f25702047f032c"
20:01:28.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:01:08 on dist-test-slave-1555"
20:01:28.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:28.632154 17018 fs_report.cc:389] FS layout report
20:01:28.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:01:28.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/wal
20:01:28.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/wal
20:01:28.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/data/data
20:01:28.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:01:28.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:01:28.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:01:28.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:01:28.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:01:28.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:01:28.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:01:28.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:01:28.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:01:28.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:01:28.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:01:28.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:28.697574 17018 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:01:28.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:28.704232 17018 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:01:28.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:28.704810 17018 kserver.cc:163] Server-wide thread pool size limit: 3276
20:01:28.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:28.834071 17018 rpc_server.cc:307] RPC server started. Bound to: 127.15.168.60:45671
20:01:28.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:28.834126 17134 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.168.60:45671 every 8 connection(s)
20:01:28.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:28.837769 17018 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/data/info.pb
20:01:28.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:28.841449 16032 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_muOGM/build/tsan/bin/kudu as pid 17018
20:01:28.842 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:28.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:28.858053 17135 sys_catalog.cc:263] Verifying existing consensus state
20:01:28.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:28.867213 17135 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c: Bootstrap starting.
20:01:28.903 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.15.168.61:32863(127.15.168.61:32863)] connecting to peer
20:01:28.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:28 dist-test-slave-1555 krb5kdc[16044](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539283, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.61@KRBTEST.COM for kudu/127.15.168.60@KRBTEST.COM
20:01:28.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:28.951177 17135 log.cc:826] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c: Log is configured to *not* fsync() on all Append() calls
20:01:28.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:28.982653 17135 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:01:28.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:28.983760 17135 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c: Bootstrap complete.
20:01:29.019 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.15.168.62:38531(127.15.168.62:38531)] connecting to peer
20:01:29.029 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.15.168.60:45671(127.15.168.60:45671)] connecting to peer
20:01:29.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:29.033610 16820 master_runner.cc:187] Error getting master registration for 127.15.168.60:45671: OK, instance_id { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" instance_seqno: 1790539288783624 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:01:29.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:29.034430 16820 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:01:29.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:29.035398 17135 raft_consensus.cc:359] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } }
20:01:29.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:29.039126 17135 raft_consensus.cc:740] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 33ab2c6f07d14efa88f25702047f032c, State: Initialized, Role: FOLLOWER
20:01:29.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:29.040369 17135 consensus_queue.cc:260] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } }
20:01:29.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:29.052211 17145 sys_catalog.cc:455] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } } }
20:01:29.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:29.053061 17145 sys_catalog.cc:458] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [sys.catalog]: This master's current role is: FOLLOWER
20:01:29.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:29.054431 17135 sys_catalog.cc:565] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [sys.catalog]: configured and running, proceeding with master startup.
20:01:29.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:29.055087 17108 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [term 1 FOLLOWER]: Advancing to term 2
20:01:29.065 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.15.168.61:32863(127.15.168.61:32863)] Successfully connected to peer
20:01:29.066 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.15.168.62:38531(127.15.168.62:38531)] Successfully connected to peer
20:01:29.065 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.15.168.60:45671(127.15.168.60:45671)] Successfully connected to peer
20:01:29.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:29.093396 16807 heartbeater.cc:499] Master 127.15.168.61:32863 was elected leader, sending a full tablet report...
20:01:29.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:29.130277 16510 heartbeater.cc:499] Master 127.15.168.61:32863 was elected leader, sending a full tablet report...
20:01:29.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:29 dist-test-slave-1555 krb5kdc[16044](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539284, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.1@KRBTEST.COM for kudu/127.15.168.60@KRBTEST.COM
20:01:29.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:29.288288 17145 sys_catalog.cc:455] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4b00615a9e9244d2a54e01564420e235. Latest consensus state: current_term: 2 leader_uuid: "4b00615a9e9244d2a54e01564420e235" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } } }
20:01:29.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:29.289005 17145 sys_catalog.cc:458] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [sys.catalog]: This master's current role is: FOLLOWER
20:01:29.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:29.289548 17145 sys_catalog.cc:455] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "4b00615a9e9244d2a54e01564420e235" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } } }
20:01:29.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:29.290172 17145 sys_catalog.cc:458] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [sys.catalog]: This master's current role is: FOLLOWER
20:01:29.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:29.299077 16806 heartbeater.cc:344] Connected to a master server at 127.15.168.60:45671
20:01:29.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:29.299588 16806 heartbeater.cc:499] Master 127.15.168.60:45671 was elected leader, sending a full tablet report...
20:01:29.302 [WARN - kudu-nio-1] (SecurityContext.java:206) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
20:01:29.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:29.328708 16509 heartbeater.cc:344] Connected to a master server at 127.15.168.60:45671
20:01:29.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:29.335673 17098 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "cd0cbe577a5e4bce9d77779ab1e537e4" instance_seqno: 1790539274711308) as {username='kudu', principal='kudu/127.15.168.1@KRBTEST.COM'} at 127.15.168.1:39839; Asking this server to re-register.
20:01:29.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:29.337755 16509 heartbeater.cc:461] Registering TS with master...
20:01:29.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:29.342195 17096 ts_manager.cc:194] Registered new tserver with Master: cd0cbe577a5e4bce9d77779ab1e537e4 (127.15.168.1:36977)
20:01:29.356 [DEBUG - kudu-nio-2] (Negotiator.java:528) SASL mechanism GSSAPI chosen for peer 127.15.168.60
20:01:29.360 [DEBUG - kudu-nio-1] (Negotiator.java:528) SASL mechanism GSSAPI chosen for peer 127.15.168.62
20:01:29.356 [DEBUG - kudu-nio-0] (Negotiator.java:528) SASL mechanism GSSAPI chosen for peer 127.15.168.61
20:01:29.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:29 dist-test-slave-1555 krb5kdc[16044](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539288, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.60@KRBTEST.COM for kudu/127.15.168.62@KRBTEST.COM
20:01:29.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:29.411235 17164 catalog_manager.cc:1308] Loaded cluster ID: d6c21a38473c4d4189831994cb48f2ce
20:01:29.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:29.431493 17164 catalog_manager.cc:1601] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c: loading cluster ID for follower catalog manager: success
20:01:29.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:29.489732 17164 catalog_manager.cc:1623] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c: acquiring CA information for follower catalog manager: success
20:01:29.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:29 dist-test-slave-1555 krb5kdc[16044](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539288, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.60@KRBTEST.COM for kudu/127.15.168.61@KRBTEST.COM
20:01:29.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:29.507138 17131 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "4b00615a9e9244d2a54e01564420e235" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } } }
20:01:29.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:29.508819 17131 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [sys.catalog]: This master's current role is: LEADER
20:01:29.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:29.565238 17164 catalog_manager.cc:1651] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:01:29.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:29.712525 16659 heartbeater.cc:499] Master 127.15.168.61:32863 was elected leader, sending a full tablet report...
20:01:29.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:29.779338 16658 heartbeater.cc:344] Connected to a master server at 127.15.168.60:45671
20:01:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:29.782413 17096 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "48e20487653845ea968ae05270a44097" instance_seqno: 1790539277425272) as {username='kudu', principal='kudu/127.15.168.2@KRBTEST.COM'} at 127.15.168.2:33529; Asking this server to re-register.
20:01:29.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:29.786207 16658 heartbeater.cc:461] Registering TS with master...
20:01:29.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:29.789314 17096 ts_manager.cc:194] Registered new tserver with Master: 48e20487653845ea968ae05270a44097 (127.15.168.2:39021)
20:01:30.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:30 dist-test-slave-1555 krb5kdc[16044](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1790539281, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.15.168.61@KRBTEST.COM
20:01:30.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:30 dist-test-slave-1555 krb5kdc[16044](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1790539281, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.15.168.62@KRBTEST.COM
20:01:30.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:30 dist-test-slave-1555 krb5kdc[16044](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1790539281, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.15.168.60@KRBTEST.COM
20:01:30.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:30 dist-test-slave-1555 krb5kdc[16044](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539290, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.3@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:30.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:30.070329 16689 init.cc:295] Successfully reacquired a new kerberos TGT
20:01:30.116 [DEBUG - kudu-nio-1] (Negotiator.java:918) Authenticated connection [id: 0x17ff391e, L:/127.0.0.1:45532 - R:/127.15.168.62:38531] using SASL/GSSAPI
20:01:30.116 [DEBUG - kudu-nio-0] (Negotiator.java:918) Authenticated connection [id: 0x6758036d, L:/127.0.0.1:43542 - R:/127.15.168.61:32863] using SASL/GSSAPI
20:01:30.126 [DEBUG - kudu-nio-2] (Negotiator.java:918) Authenticated connection [id: 0xc461c4fb, L:/127.0.0.1:52758 - R:/127.15.168.60:45671] using SASL/GSSAPI
20:01:30.178 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
20:01:30.179 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=1333), Traces: [0ms] refreshing cache from master, [37ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.61:32863, [168ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.62:38531, [178ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.168.60:45671, [1296ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.62:38531: OK, [1296ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.60:45671: OK, [1297ms] Sub RPC ConnectToMaster: received response from server master-127.15.168.61:32863: OK, deferred=null) after lookup
20:01:30.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:30.303774 16806 heartbeater.cc:499] Master 127.15.168.60:45671 was elected leader, sending a full tablet report...
20:01:30.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:30.306658 17096 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "83fbdd41d37846738f806b86554e3f41" instance_seqno: 1790539280611250) as {username='kudu', principal='kudu/127.15.168.3@KRBTEST.COM'} at 127.15.168.3:51995; Asking this server to re-register.
20:01:30.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:30.308518 16806 heartbeater.cc:461] Registering TS with master...
20:01:30.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:30.311821 17096 ts_manager.cc:194] Registered new tserver with Master: 83fbdd41d37846738f806b86554e3f41 (127.15.168.3:35707)
20:01:31.203 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
20:01:31.203 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
20:01:31.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:31 dist-test-slave-1555 krb5kdc[16044](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539291, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:31.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:01:31.242 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:31.242 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/krb5cc
20:01:31.244 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:01:31.244 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.15.168.61:32863
20:01:31.245 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
20:01:31.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:31.245649 16032 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_muOGM/build/tsan/bin/kudu with pid 16820
20:01:31.289 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:31.289 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.15.168.62:38531
20:01:31.289 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
20:01:31.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:31.290096 16032 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_muOGM/build/tsan/bin/kudu with pid 16910
20:01:31.298 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.15.168.61:32863(127.15.168.61:32863)] handling channelInactive
20:01:31.299 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.168.61:32863(127.15.168.61:32863)] cleaning up while in state READY due to: connection closed
20:01:31.329 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.15.168.62:38531(127.15.168.62:38531)] handling channelInactive
20:01:31.329 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.168.62:38531(127.15.168.62:38531)] cleaning up while in state READY due to: connection closed
20:01:31.331 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:31.332 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.15.168.60:45671
20:01:31.332 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
20:01:31.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:31.332944 16032 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_muOGM/build/tsan/bin/kudu with pid 17018
20:01:31.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:31.363060 16398 connection.cc:573] client connection to 127.15.168.60:45671 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
20:01:31.364 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.15.168.60:45671(127.15.168.60:45671)] handling channelInactive
20:01:31.364 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:31.364 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.15.168.60:45671(127.15.168.60:45671)] cleaning up while in state READY due to: connection closed
20:01:31.365 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.15.168.61:32863
20:01:31.365 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
20:01:31.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:31.366648 16032 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_muOGM/build/tsan/bin/kudu
20:01:31.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_muOGM/build/tsan/bin/kudu
20:01:31.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/wal
20:01:31.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/data
20:01:31.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:01:31.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:01:31.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:01:31.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:31.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:31.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:31.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/logs
20:01:31.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/data/info.pb
20:01:31.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:31.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:01:31.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:01:31.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:01:31.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:01:31.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:31.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:31.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:31.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:01:31.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:01:31.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:31.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:31.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.168.61:32863
20:01:31.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.168.61
20:01:31.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35057
20:01:31.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab
20:01:31.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.168.61
20:01:31.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:31.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:01:31.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:01:31.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:01:31.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.168.20:32967
20:01:31.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:31.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:31.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:31.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:31.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.168.62:38531,127.15.168.61:32863,127.15.168.60:45671
20:01:31.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:31.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41913/jwks.json
20:01:31.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:01:31.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:01:31.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:31.722628 17178 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:01:31.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:31.723452 17178 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:01:31.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:31.724615 17178 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:01:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:31.763895 17178 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:01:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:31.764194 17178 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:01:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:31.764482 17178 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:01:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:31.764696 17178 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:01:31.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:31.764906 17178 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:01:31.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:31.765105 17178 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:01:31.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:31.765254 17178 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:41913/jwks.json
20:01:31.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:31.805763 17178 master_runner.cc:387] Master server non-default flags:
20:01:31.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/logs
20:01:31.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:31.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:01:31.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:31.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.168.20:32967
20:01:31.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:31.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:31.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/data
20:01:31.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/wal
20:01:31.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:01:31.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:31.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.168.62:38531,127.15.168.61:32863,127.15.168.60:45671
20:01:31.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab
20:01:31.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.168.61
20:01:31.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:31.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:31.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:31.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.168.61:32863
20:01:31.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:31.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:01:31.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:31.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41913/jwks.json
20:01:31.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:31.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:01:31.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:01:31.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:31.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:31.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/data/info.pb
20:01:31.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.168.61
20:01:31.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35057
20:01:31.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:01:31.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:31.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:01:31.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:01:31.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:31.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:01:31.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:01:31.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:01:31.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3b18f0139b1d21a072b7353269301fba16dca54e
20:01:31.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:01:31.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Sep 2026 19:43:26 UTC on e7f111948823
20:01:31.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14030
20:01:31.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:01:31.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:31.807169 17178 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:01:31.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:31.808223 17178 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:01:31.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:31.825133 17187 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:33.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:33.225085 17185 debug-util.cc:398] Leaking SignalData structure 0x7b0800037cc0 after lost signal to thread 17178
20:01:33.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:33.237754 17185 kernel_stack_watchdog.cc:198] Thread 17178 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 394ms:
20:01:33.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:01:33.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:01:33.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:01:33.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:01:33.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:31.825315 17186 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:33.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:33.243729 17188 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1413 milliseconds
20:01:33.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:33.244033 17178 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.418s user 0.000s sys 0.002s
20:01:33.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:33.244478 17178 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.419s user 0.000s sys 0.003s
20:01:33.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:33.246270 17190 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:33.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:33.246333 17178 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:01:33.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:33.247639 17178 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:01:33.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:33.250689 17178 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:01:33.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:33.252192 17178 hybrid_clock.cc:648] HybridClock initialized: now 1790539293252150 us; error 48 us; skew 500 ppm
20:01:33.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:33.253105 17178 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:01:33.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:33 dist-test-slave-1555 krb5kdc[16044](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539293, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:33.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:33.259361 17178 init.cc:377] Logged in from keytab as kudu/127.15.168.61@KRBTEST.COM (short username kudu)
20:01:33.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:33.264942 17178 webserver.cc:533] Webserver started at http://127.15.168.61:35057/ using document root <none> and password file <none>
20:01:33.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:33.265861 17178 fs_manager.cc:362] Metadata directory not provided
20:01:33.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:33.266052 17178 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:01:33.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:33.274294 17178 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.005s sys 0.000s
20:01:33.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:33.278860 17206 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:01:33.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:33.280216 17178 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
20:01:33.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:33.280529 17178 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/data,/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/wal
20:01:33.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4b00615a9e9244d2a54e01564420e235"
20:01:33.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:01:06 on dist-test-slave-1555"
20:01:33.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:33.282395 17178 fs_report.cc:389] FS layout report
20:01:33.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:01:33.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/wal
20:01:33.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/wal
20:01:33.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/data/data
20:01:33.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:01:33.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:01:33.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:01:33.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:01:33.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:01:33.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:01:33.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:01:33.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:01:33.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:01:33.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:01:33.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:01:33.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:33.380486 17178 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:01:33.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:33.391184 17178 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:01:33.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:33.391821 17178 kserver.cc:163] Server-wide thread pool size limit: 3276
20:01:33.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:33.479537 17178 rpc_server.cc:307] RPC server started. Bound to: 127.15.168.61:32863
20:01:33.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:33.479576 17257 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.168.61:32863 every 8 connection(s)
20:01:33.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:33.483479 17178 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/data/info.pb
20:01:33.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:33.492144 16032 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_muOGM/build/tsan/bin/kudu as pid 17178
20:01:33.493 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:33.493 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.15.168.62:38531
20:01:33.494 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
20:01:33.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:33.495496 16032 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_muOGM/build/tsan/bin/kudu
20:01:33.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_muOGM/build/tsan/bin/kudu
20:01:33.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/wal
20:01:33.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/data
20:01:33.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:01:33.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:01:33.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:01:33.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:33.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:33.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:33.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/logs
20:01:33.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/data/info.pb
20:01:33.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:33.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:01:33.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:01:33.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:01:33.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:01:33.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:33.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:33.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:33.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:01:33.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:01:33.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:33.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:33.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.168.62:38531
20:01:33.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.168.62
20:01:33.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41983
20:01:33.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab
20:01:33.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.168.62
20:01:33.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:33.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:01:33.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:01:33.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:01:33.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.168.20:32967
20:01:33.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:33.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:33.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:33.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:33.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.168.62:38531,127.15.168.61:32863,127.15.168.60:45671
20:01:33.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:33.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41913/jwks.json
20:01:33.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:01:33.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:01:33.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:33.495973 17258 sys_catalog.cc:263] Verifying existing consensus state
20:01:33.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:33.501535 17258 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235: Bootstrap starting.
20:01:33.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:33.545856 17258 log.cc:826] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235: Log is configured to *not* fsync() on all Append() calls
20:01:33.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:33.564291 17258 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:01:33.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:33.565166 17258 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235: Bootstrap complete.
20:01:33.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:33.588560 17258 raft_consensus.cc:359] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } }
20:01:33.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:33.590772 17258 raft_consensus.cc:740] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4b00615a9e9244d2a54e01564420e235, State: Initialized, Role: FOLLOWER
20:01:33.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:33.591888 17258 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } }
20:01:33.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:33.597523 17262 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } } }
20:01:33.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:33.598518 17262 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [sys.catalog]: This master's current role is: FOLLOWER
20:01:33.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:33.598949 17258 sys_catalog.cc:565] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [sys.catalog]: configured and running, proceeding with master startup.
20:01:33.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:33.631912 17273 catalog_manager.cc:1308] Loaded cluster ID: d6c21a38473c4d4189831994cb48f2ce
20:01:33.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:33.632637 17273 catalog_manager.cc:1601] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235: loading cluster ID for follower catalog manager: success
20:01:33.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:33.641268 17273 catalog_manager.cc:1623] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235: acquiring CA information for follower catalog manager: success
20:01:33.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:33.649984 17273 catalog_manager.cc:1651] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:01:33.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:33.637566 17178 master_runner.cc:187] Error getting master registration for 127.15.168.62:38531: Network error: Client connection negotiation failed: client connection to 127.15.168.62:38531: connect: Connection refused (error 111),
20:01:33.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:33.660889 17178 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:01:33.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:33.795451 16659 heartbeater.cc:344] Connected to a master server at 127.15.168.61:32863
20:01:33.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:33.802127 17223 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "48e20487653845ea968ae05270a44097" instance_seqno: 1790539277425272) as {username='kudu', principal='kudu/127.15.168.2@KRBTEST.COM'} at 127.15.168.2:58627; Asking this server to re-register.
20:01:33.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:33.804092 16659 heartbeater.cc:461] Registering TS with master...
20:01:33.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:33.807703 17223 ts_manager.cc:194] Registered new tserver with Master: 48e20487653845ea968ae05270a44097 (127.15.168.2:39021)
20:01:33.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:33.915436 17260 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:01:33.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:33.916277 17260 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:01:33.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:33.917408 17260 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:01:33.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:33.965205 17260 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:01:33.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:33.965534 17260 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:01:33.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:33.965914 17260 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:01:33.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:33.966212 17260 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:01:33.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:33.966434 17260 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:01:33.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:33.966630 17260 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:01:33.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:33.966779 17260 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:41913/jwks.json
20:01:34.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:34.017233 17260 master_runner.cc:387] Master server non-default flags:
20:01:34.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/logs
20:01:34.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:34.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:01:34.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:34.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.168.20:32967
20:01:34.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:34.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:34.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/data
20:01:34.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/wal
20:01:34.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:01:34.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:34.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.168.62:38531,127.15.168.61:32863,127.15.168.60:45671
20:01:34.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab
20:01:34.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.168.62
20:01:34.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:34.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:34.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:34.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.168.62:38531
20:01:34.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:34.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:01:34.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:34.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41913/jwks.json
20:01:34.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:34.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:01:34.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:01:34.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:34.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:34.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/data/info.pb
20:01:34.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.168.62
20:01:34.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41983
20:01:34.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:01:34.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:34.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:01:34.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:01:34.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:34.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:01:34.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:01:34.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:01:34.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3b18f0139b1d21a072b7353269301fba16dca54e
20:01:34.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:01:34.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Sep 2026 19:43:26 UTC on e7f111948823
20:01:34.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14030
20:01:34.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:01:34.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:34.018558 17260 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:01:34.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:34.019511 17260 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:01:34.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:34.031498 17284 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:34.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:34.200917 16807 heartbeater.cc:344] Connected to a master server at 127.15.168.61:32863
20:01:34.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:34.204876 17223 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "83fbdd41d37846738f806b86554e3f41" instance_seqno: 1790539280611250) as {username='kudu', principal='kudu/127.15.168.3@KRBTEST.COM'} at 127.15.168.3:37897; Asking this server to re-register.
20:01:34.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:34.206657 16807 heartbeater.cc:461] Registering TS with master...
20:01:34.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:34.210695 17223 ts_manager.cc:194] Registered new tserver with Master: 83fbdd41d37846738f806b86554e3f41 (127.15.168.3:35707)
20:01:34.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:34.250384 16510 heartbeater.cc:344] Connected to a master server at 127.15.168.61:32863
20:01:34.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:34.254329 17223 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "cd0cbe577a5e4bce9d77779ab1e537e4" instance_seqno: 1790539274711308) as {username='kudu', principal='kudu/127.15.168.1@KRBTEST.COM'} at 127.15.168.1:53935; Asking this server to re-register.
20:01:34.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:34.256002 16510 heartbeater.cc:461] Registering TS with master...
20:01:34.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:34.260186 17223 ts_manager.cc:194] Registered new tserver with Master: cd0cbe577a5e4bce9d77779ab1e537e4 (127.15.168.1:36977)
20:01:34.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:34 dist-test-slave-1555 krb5kdc[16044](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539294, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.1@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:34.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:34.413129 16390 init.cc:295] Successfully reacquired a new kerberos TGT
20:01:34.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:34.859513 17299 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:01:34.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:34.860728 17299 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } }
20:01:34.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:34.889514 17299 leader_election.cc:290] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 95585e2dbd5a475f9466ce816f5fffff (127.15.168.62:38531), 33ab2c6f07d14efa88f25702047f032c (127.15.168.60:45671)
20:01:34.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:34.903810 17209 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.168.60:45671: connect: Connection refused (error 111)
20:01:34.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:34.926888 17209 leader_election.cc:336] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 33ab2c6f07d14efa88f25702047f032c (127.15.168.60:45671): Network error: Client connection negotiation failed: client connection to 127.15.168.60:45671: connect: Connection refused (error 111)
20:01:34.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:34.944835 15456 debug-util.cc:398] Leaking SignalData structure 0x7b08000b0b80 after lost signal to thread 15391
20:01:34.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:34.945785 15456 debug-util.cc:398] Leaking SignalData structure 0x7b08000b1ae0 after lost signal to thread 15459
20:01:35.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:35.012809 17210 leader_election.cc:336] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 95585e2dbd5a475f9466ce816f5fffff (127.15.168.62:38531): Network error: Client connection negotiation failed: client connection to 127.15.168.62:38531: connect: Connection refused (error 111)
20:01:35.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:35.013310 17210 leader_election.cc:304] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4b00615a9e9244d2a54e01564420e235; no voters: 33ab2c6f07d14efa88f25702047f032c, 95585e2dbd5a475f9466ce816f5fffff
20:01:35.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:35.014633 17299 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
20:01:36.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:36.008612 15734 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad4c0 after lost signal to thread 15606
20:01:36.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:36.009703 15734 debug-util.cc:398] Leaking SignalData structure 0x7b08000b6c60 after lost signal to thread 15737
20:01:36.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:35.433344 17283 debug-util.cc:398] Leaking SignalData structure 0x7b0800037cc0 after lost signal to thread 17260
20:01:36.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:36.052192 17260 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.019s user 0.621s sys 1.013s
20:01:36.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:36.052582 17260 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.020s user 0.621s sys 1.013s
20:01:36.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:36.053766 17286 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2019 milliseconds
20:01:36.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:36.056494 17287 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:36.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:36.056437 17260 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:01:36.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:34.033792 17285 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:36.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:36.062143 17260 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:01:36.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:36.075763 17260 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:01:36.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:36.083591 17260 hybrid_clock.cc:648] HybridClock initialized: now 1790539296083537 us; error 63 us; skew 500 ppm
20:01:36.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:36.084888 17260 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:01:36.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:36 dist-test-slave-1555 krb5kdc[16044](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539296, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:36.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:36.093950 17260 init.cc:377] Logged in from keytab as kudu/127.15.168.62@KRBTEST.COM (short username kudu)
20:01:36.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:36.139875 17260 webserver.cc:533] Webserver started at http://127.15.168.62:41983/ using document root <none> and password file <none>
20:01:36.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:36.141175 17260 fs_manager.cc:362] Metadata directory not provided
20:01:36.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:36.141484 17260 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:01:36.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:36.171079 17260 fs_manager.cc:714] Time spent opening directory manager: real 0.027s user 0.012s sys 0.012s
20:01:36.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:36.187933 17315 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:01:36.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:36.189690 17260 fs_manager.cc:730] Time spent opening block manager: real 0.015s user 0.006s sys 0.008s
20:01:36.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:36.190121 17260 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/data,/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/wal
20:01:36.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "95585e2dbd5a475f9466ce816f5fffff"
20:01:36.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:01:04 on dist-test-slave-1555"
20:01:36.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:36.193220 17260 fs_report.cc:389] FS layout report
20:01:36.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:01:36.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/wal
20:01:36.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/wal
20:01:36.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/data/data
20:01:36.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:01:36.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:01:36.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:01:36.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:01:36.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:01:36.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:01:36.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:01:36.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:01:36.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:01:36.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:01:36.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:01:36.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:36.302052 17260 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:01:36.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:36.310500 17260 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:01:36.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:36.311422 17260 kserver.cc:163] Server-wide thread pool size limit: 3276
20:01:36.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:36.707931 17369 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.168.62:38531 every 8 connection(s)
20:01:36.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:36.708547 17260 rpc_server.cc:307] RPC server started. Bound to: 127.15.168.62:38531
20:01:36.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:36.713122 17260 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/data/info.pb
20:01:36.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:36.715896 16032 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_muOGM/build/tsan/bin/kudu as pid 17260
20:01:36.717 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:36.717 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.15.168.60:45671
20:01:36.717 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
20:01:36.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:36.719754 16032 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_muOGM/build/tsan/bin/kudu
20:01:36.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_muOGM/build/tsan/bin/kudu
20:01:36.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/wal
20:01:36.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/data
20:01:36.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:01:36.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:01:36.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:01:36.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:36.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:36.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:36.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/logs
20:01:36.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/data/info.pb
20:01:36.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:36.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:01:36.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:01:36.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:01:36.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:01:36.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:36.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:36.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:36.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:01:36.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:01:36.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:36.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:36.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.168.60:45671
20:01:36.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.168.60
20:01:36.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33645
20:01:36.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab
20:01:36.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.168.60
20:01:36.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:36.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:01:36.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:01:36.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:01:36.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.168.20:32967
20:01:36.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:36.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:36.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:36.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:36.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.168.62:38531,127.15.168.61:32863,127.15.168.60:45671
20:01:36.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:36.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41913/jwks.json
20:01:36.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:01:36.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:01:36.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:36.739749 17370 sys_catalog.cc:263] Verifying existing consensus state
20:01:36.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:36.748819 17370 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff: Bootstrap starting.
20:01:36.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:36.792027 17373 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:01:36.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:36.793052 17373 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } }
20:01:36.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:36.809252 17373 leader_election.cc:290] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 95585e2dbd5a475f9466ce816f5fffff (127.15.168.62:38531), 33ab2c6f07d14efa88f25702047f032c (127.15.168.60:45671)
20:01:36.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:36.827770 17209 leader_election.cc:336] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 33ab2c6f07d14efa88f25702047f032c (127.15.168.60:45671): Network error: Client connection negotiation failed: client connection to 127.15.168.60:45671: connect: Connection refused (error 111)
20:01:36.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:36.829128 17370 log.cc:826] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff: Log is configured to *not* fsync() on all Append() calls
20:01:36.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:36 dist-test-slave-1555 krb5kdc[16044](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539293, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.61@KRBTEST.COM for kudu/127.15.168.62@KRBTEST.COM
20:01:36.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:36 dist-test-slave-1555 krb5kdc[16044](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539287, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.2@KRBTEST.COM for kudu/127.15.168.62@KRBTEST.COM
20:01:36.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:36.972500 16660 heartbeater.cc:344] Connected to a master server at 127.15.168.62:38531
20:01:36.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:36.987295 17370 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:01:36.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:36.990347 17370 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff: Bootstrap complete.
20:01:37.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.024660 17370 raft_consensus.cc:359] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } }
20:01:37.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.038272 17370 raft_consensus.cc:740] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 95585e2dbd5a475f9466ce816f5fffff, State: Initialized, Role: FOLLOWER
20:01:37.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.041435 17370 consensus_queue.cc:260] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } }
20:01:37.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.057132 17344 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4b00615a9e9244d2a54e01564420e235" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "95585e2dbd5a475f9466ce816f5fffff" is_pre_election: true
20:01:37.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.058099 17344 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4b00615a9e9244d2a54e01564420e235 in term 2.
20:01:37.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.060117 17210 leader_election.cc:304] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4b00615a9e9244d2a54e01564420e235, 95585e2dbd5a475f9466ce816f5fffff; no voters: 33ab2c6f07d14efa88f25702047f032c
20:01:37.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.060388 17370 sys_catalog.cc:565] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [sys.catalog]: configured and running, proceeding with master startup.
20:01:37.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.061312 17373 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [term 2 FOLLOWER]: Leader pre-election won for term 3
20:01:37.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.061635 17373 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:01:37.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.061976 17373 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [term 2 FOLLOWER]: Advancing to term 3
20:01:37.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.065420 17383 sys_catalog.cc:455] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } } }
20:01:37.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.066243 17383 sys_catalog.cc:458] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [sys.catalog]: This master's current role is: FOLLOWER
20:01:37.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.079411 17373 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } }
20:01:37.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.085467 17344 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4b00615a9e9244d2a54e01564420e235" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "95585e2dbd5a475f9466ce816f5fffff"
20:01:37.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.086053 17344 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [term 2 FOLLOWER]: Advancing to term 3
20:01:37.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:37.093894 17209 leader_election.cc:336] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 33ab2c6f07d14efa88f25702047f032c (127.15.168.60:45671): Network error: Client connection negotiation failed: client connection to 127.15.168.60:45671: connect: Connection refused (error 111)
20:01:37.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.094900 17373 leader_election.cc:290] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [CANDIDATE]: Term 3 election: Requested vote from peers 95585e2dbd5a475f9466ce816f5fffff (127.15.168.62:38531), 33ab2c6f07d14efa88f25702047f032c (127.15.168.60:45671)
20:01:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.096158 17344 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4b00615a9e9244d2a54e01564420e235 in term 3.
20:01:37.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.097244 17210 leader_election.cc:304] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4b00615a9e9244d2a54e01564420e235, 95585e2dbd5a475f9466ce816f5fffff; no voters: 33ab2c6f07d14efa88f25702047f032c
20:01:37.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.098843 17373 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [term 3 FOLLOWER]: Leader election won for term 3
20:01:37.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.108903 17373 raft_consensus.cc:697] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [term 3 LEADER]: Becoming Leader. State: Replica: 4b00615a9e9244d2a54e01564420e235, State: Running, Role: LEADER
20:01:37.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.110486 17373 consensus_queue.cc:237] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } }
20:01:37.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.134800 17386 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4b00615a9e9244d2a54e01564420e235. Latest consensus state: current_term: 3 leader_uuid: "4b00615a9e9244d2a54e01564420e235" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } } }
20:01:37.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.136328 17386 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [sys.catalog]: This master's current role is: LEADER
20:01:37.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.155210 17391 catalog_manager.cc:1524] Loading table and tablet metadata into memory...
20:01:37.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.172531 17391 catalog_manager.cc:1533] Initializing Kudu cluster ID...
20:01:37.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.185096 17391 catalog_manager.cc:1308] Loaded cluster ID: d6c21a38473c4d4189831994cb48f2ce
20:01:37.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.188485 17391 catalog_manager.cc:1544] Initializing Kudu internal certificate authority...
20:01:37.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.195876 17397 catalog_manager.cc:818] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
20:01:37.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.199642 17391 catalog_manager.cc:1553] Loading token signing keys...
20:01:37.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:37 dist-test-slave-1555 krb5kdc[16044](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539297, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.2@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:37.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.206434 16540 init.cc:295] Successfully reacquired a new kerberos TGT
20:01:37.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.204947 17391 catalog_manager.cc:6230] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235: Loaded TSK: 0
20:01:37.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.220471 17391 catalog_manager.cc:1563] Initializing in-progress tserver states...
20:01:37.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:37 dist-test-slave-1555 krb5kdc[16044](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539296, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.62@KRBTEST.COM for kudu/127.15.168.62@KRBTEST.COM
20:01:37.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:37 dist-test-slave-1555 krb5kdc[16044](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539290, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.3@KRBTEST.COM for kudu/127.15.168.62@KRBTEST.COM
20:01:37.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.275950 16812 heartbeater.cc:344] Connected to a master server at 127.15.168.62:38531
20:01:37.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.283491 17334 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "83fbdd41d37846738f806b86554e3f41" instance_seqno: 1790539280611250) as {username='kudu', principal='kudu/127.15.168.3@KRBTEST.COM'} at 127.15.168.3:60151; Asking this server to re-register.
20:01:37.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.285708 16812 heartbeater.cc:461] Registering TS with master...
20:01:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.291107 17334 ts_manager.cc:194] Registered new tserver with Master: 83fbdd41d37846738f806b86554e3f41 (127.15.168.3:35707)
20:01:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:37 dist-test-slave-1555 krb5kdc[16044](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539296, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.62@KRBTEST.COM for kudu/127.15.168.61@KRBTEST.COM
20:01:37.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:37 dist-test-slave-1555 krb5kdc[16044](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539294, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.1@KRBTEST.COM for kudu/127.15.168.62@KRBTEST.COM
20:01:37.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.396361 16511 heartbeater.cc:344] Connected to a master server at 127.15.168.62:38531
20:01:37.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.399271 17334 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "cd0cbe577a5e4bce9d77779ab1e537e4" instance_seqno: 1790539274711308) as {username='kudu', principal='kudu/127.15.168.1@KRBTEST.COM'} at 127.15.168.1:37689; Asking this server to re-register.
20:01:37.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.400627 16511 heartbeater.cc:461] Registering TS with master...
20:01:37.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.403590 17334 ts_manager.cc:194] Registered new tserver with Master: cd0cbe577a5e4bce9d77779ab1e537e4 (127.15.168.1:36977)
20:01:37.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.338590 17260 master_runner.cc:187] Error getting master registration for 127.15.168.60:45671: Network error: Client connection negotiation failed: client connection to 127.15.168.60:45671: connect: Connection refused (error 111),
20:01:37.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.419533 17260 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:01:37.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:37.556583 17209 consensus_peers.cc:597] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 -> Peer 33ab2c6f07d14efa88f25702047f032c (127.15.168.60:45671): Couldn't send request to peer 33ab2c6f07d14efa88f25702047f032c. Status: Network error: Client connection negotiation failed: client connection to 127.15.168.60:45671: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:01:37.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:37.715145 17372 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:01:37.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:37.716154 17372 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:01:37.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:37.717509 17372 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:01:37.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.738082 17344 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [term 3 FOLLOWER]: Refusing update from remote peer 4b00615a9e9244d2a54e01564420e235: Log matching property violated. Preceding OpId in replica: term: 2 index: 5. Preceding OpId from leader: term: 3 index: 6. (index mismatch)
20:01:37.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.739965 17386 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [LEADER]: Connected to new peer: Peer: permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
20:01:37.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:37.789965 17372 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:01:37.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:37.790335 17372 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:01:37.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:37.790892 17372 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:01:37.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:37.791280 17372 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:01:37.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:37.791679 17372 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:01:37.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:37.792026 17372 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:01:37.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:37.792311 17372 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:41913/jwks.json
20:01:37.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.777915 17407 sys_catalog.cc:455] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4b00615a9e9244d2a54e01564420e235. Latest consensus state: current_term: 3 leader_uuid: "4b00615a9e9244d2a54e01564420e235" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } } }
20:01:37.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.793987 17407 sys_catalog.cc:458] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [sys.catalog]: This master's current role is: FOLLOWER
20:01:37.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.827689 17407 sys_catalog.cc:455] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "4b00615a9e9244d2a54e01564420e235" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } } }
20:01:37.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.828472 17407 sys_catalog.cc:458] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [sys.catalog]: This master's current role is: FOLLOWER
20:01:37.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.849763 17412 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "4b00615a9e9244d2a54e01564420e235" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } } }
20:01:37.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.850569 17412 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [sys.catalog]: This master's current role is: LEADER
20:01:37.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.883531 17372 master_runner.cc:387] Master server non-default flags:
20:01:37.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/logs
20:01:37.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:37.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:01:37.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:37.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.168.20:32967
20:01:37.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:37.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:37.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/data
20:01:37.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/wal
20:01:37.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:01:37.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:37.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.168.62:38531,127.15.168.61:32863,127.15.168.60:45671
20:01:37.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab
20:01:37.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.168.60
20:01:37.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:37.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:37.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:37.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.168.60:45671
20:01:37.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:37.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:01:37.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:37.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41913/jwks.json
20:01:37.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:37.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:01:37.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:01:37.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:37.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:37.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/data/info.pb
20:01:37.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.168.60
20:01:37.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33645
20:01:37.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:01:37.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:37.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:01:37.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:01:37.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:37.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:01:37.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:01:37.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:01:37.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3b18f0139b1d21a072b7353269301fba16dca54e
20:01:37.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:01:37.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Sep 2026 19:43:26 UTC on e7f111948823
20:01:37.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14030
20:01:37.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:01:37.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.885488 17372 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:01:37.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.886925 17372 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:01:37.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:37.917578 17416 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:37.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.980618 17334 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "48e20487653845ea968ae05270a44097" instance_seqno: 1790539277425272) as {username='kudu', principal='kudu/127.15.168.2@KRBTEST.COM'} at 127.15.168.2:38131; Asking this server to re-register.
20:01:37.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.982328 16660 heartbeater.cc:461] Registering TS with master...
20:01:37.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:37.985810 17334 ts_manager.cc:194] Registered new tserver with Master: 48e20487653845ea968ae05270a44097 (127.15.168.2:39021)
20:01:38.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:38.232319 17397 catalog_manager.cc:1308] Loaded cluster ID: d6c21a38473c4d4189831994cb48f2ce
20:01:38.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:38.232899 17397 catalog_manager.cc:1601] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff: loading cluster ID for follower catalog manager: success
20:01:38.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:38.244352 16807 heartbeater.cc:499] Master 127.15.168.61:32863 was elected leader, sending a full tablet report...
20:01:38.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:38.258163 17397 catalog_manager.cc:1623] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff: acquiring CA information for follower catalog manager: success
20:01:38.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:38.279109 17397 catalog_manager.cc:1651] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:01:38.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:38.316825 16510 heartbeater.cc:499] Master 127.15.168.61:32863 was elected leader, sending a full tablet report...
20:01:38.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:38.710795 17178 master_runner.cc:187] Error getting master registration for 127.15.168.60:45671: Network error: Client connection negotiation failed: client connection to 127.15.168.60:45671: connect: Connection refused (error 111),
20:01:38.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:38.711411 17178 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:01:38.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:38.905745 16659 heartbeater.cc:499] Master 127.15.168.61:32863 was elected leader, sending a full tablet report...
20:01:39.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:39.164846 15536 debug-util.cc:398] Leaking SignalData structure 0x7b080007fb60 after lost signal to thread 15467
20:01:39.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:39.166041 15536 debug-util.cc:398] Leaking SignalData structure 0x7b080007fe80 after lost signal to thread 15539
20:01:39.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:39.987231 17209 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.168.60:45671: connect: Connection refused (error 111) [suppressed 8 similar messages]
20:01:39.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:39.990921 17209 consensus_peers.cc:597] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 -> Peer 33ab2c6f07d14efa88f25702047f032c (127.15.168.60:45671): Couldn't send request to peer 33ab2c6f07d14efa88f25702047f032c. Status: Network error: Client connection negotiation failed: client connection to 127.15.168.60:45671: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
20:01:40.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:40 dist-test-slave-1555 krb5kdc[16044](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539300, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.3@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:40.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:40.078164 16689 init.cc:295] Successfully reacquired a new kerberos TGT
20:01:40.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:37.936691 17420 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:40.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:37.928737 17417 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:40.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:40.412240 17418 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2483 milliseconds
20:01:40.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:40.412340 17372 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:01:40.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:40.414759 17372 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:01:40.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:40.418332 17372 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:01:40.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:40.419838 17372 hybrid_clock.cc:648] HybridClock initialized: now 1790539300419803 us; error 40 us; skew 500 ppm
20:01:40.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:40.420672 17372 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:01:40.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:40 dist-test-slave-1555 krb5kdc[16044](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539300, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:40.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:40.426977 17372 init.cc:377] Logged in from keytab as kudu/127.15.168.60@KRBTEST.COM (short username kudu)
20:01:40.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:40.434906 17372 webserver.cc:533] Webserver started at http://127.15.168.60:33645/ using document root <none> and password file <none>
20:01:40.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:40.436407 17372 fs_manager.cc:362] Metadata directory not provided
20:01:40.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:40.436865 17372 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:01:40.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:40.448318 17372 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.006s sys 0.001s
20:01:40.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:40.454414 17442 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:01:40.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:40.456372 17372 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.005s sys 0.000s
20:01:40.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:40.456768 17372 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/data,/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/wal
20:01:40.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "33ab2c6f07d14efa88f25702047f032c"
20:01:40.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:01:08 on dist-test-slave-1555"
20:01:40.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:40.459415 17372 fs_report.cc:389] FS layout report
20:01:40.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:01:40.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/wal
20:01:40.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/wal
20:01:40.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/data/data
20:01:40.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:01:40.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:01:40.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:01:40.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:01:40.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:01:40.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:01:40.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:01:40.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:01:40.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:01:40.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:01:40.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:01:40.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:40.538712 17372 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:01:40.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:40.548770 17372 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:01:40.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:40.549367 17372 kserver.cc:163] Server-wide thread pool size limit: 3276
20:01:40.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:40.645584 17372 rpc_server.cc:307] RPC server started. Bound to: 127.15.168.60:45671
20:01:40.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:40.645701 17494 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.168.60:45671 every 8 connection(s)
20:01:40.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:40.648772 17372 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/data/info.pb
20:01:40.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:40.656184 16032 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_muOGM/build/tsan/bin/kudu as pid 17372
20:01:40.657 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:40.657 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.15.168.61:32863(127.15.168.61:32863)] connecting to peer
20:01:40.660 [DEBUG - kudu-nio-3] (Connection.java:757) [peer master-127.15.168.61:32863(127.15.168.61:32863)] Successfully connected to peer
20:01:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:40.697376 17495 sys_catalog.cc:263] Verifying existing consensus state
20:01:40.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:40.713246 17495 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c: Bootstrap starting.
20:01:40.714 [DEBUG - kudu-nio-3] (Negotiator.java:528) SASL mechanism GSSAPI chosen for peer 127.15.168.61
20:01:40.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:40.792346 17495 log.cc:826] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c: Log is configured to *not* fsync() on all Append() calls
20:01:40.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:40 dist-test-slave-1555 krb5kdc[16044](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1790539291, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.15.168.61@KRBTEST.COM
20:01:40.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:40.824731 17495 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:01:40.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:40.825796 17495 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c: Bootstrap complete.
20:01:40.863 [DEBUG - kudu-nio-3] (Negotiator.java:918) Authenticated connection [id: 0xb2bca8df, L:/127.0.0.1:55062 - R:/127.15.168.61:32863] using SASL/GSSAPI
20:01:40.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:40.862069 17495 raft_consensus.cc:359] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } }
20:01:40.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:40.865525 17495 raft_consensus.cc:740] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 33ab2c6f07d14efa88f25702047f032c, State: Initialized, Role: FOLLOWER
20:01:40.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:40.866524 17495 consensus_queue.cc:260] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } }
20:01:40.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:40 dist-test-slave-1555 krb5kdc[16044](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539293, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.61@KRBTEST.COM for kudu/127.15.168.60@KRBTEST.COM
20:01:40.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:40.878086 17502 sys_catalog.cc:455] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } } }
20:01:40.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:40.879942 17495 sys_catalog.cc:565] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [sys.catalog]: configured and running, proceeding with master startup.
20:01:40.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:40.880021 17502 sys_catalog.cc:458] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [sys.catalog]: This master's current role is: FOLLOWER
20:01:40.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:40.906688 17470 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [term 2 FOLLOWER]: Advancing to term 3
20:01:40.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:40.932571 17515 sys_catalog.cc:455] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "4b00615a9e9244d2a54e01564420e235" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } } }
20:01:40.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:40.933696 17515 sys_catalog.cc:458] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [sys.catalog]: This master's current role is: FOLLOWER
20:01:40.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:40.937628 17502 sys_catalog.cc:455] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4b00615a9e9244d2a54e01564420e235. Latest consensus state: current_term: 3 leader_uuid: "4b00615a9e9244d2a54e01564420e235" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } } }
20:01:40.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:40.938146 17502 sys_catalog.cc:458] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [sys.catalog]: This master's current role is: FOLLOWER
20:01:40.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:40.959954 17516 catalog_manager.cc:1308] Loaded cluster ID: d6c21a38473c4d4189831994cb48f2ce
20:01:40.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:40.960309 17516 catalog_manager.cc:1601] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c: loading cluster ID for follower catalog manager: success
20:01:40.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:40.969969 17516 catalog_manager.cc:1623] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c: acquiring CA information for follower catalog manager: success
20:01:40.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:40 dist-test-slave-1555 krb5kdc[16044](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539300, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.60@KRBTEST.COM for kudu/127.15.168.62@KRBTEST.COM
20:01:40.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:40.984012 17516 catalog_manager.cc:1651] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:01:40.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:40.985112 16658 heartbeater.cc:344] Connected to a master server at 127.15.168.60:45671
20:01:40.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:40.992668 17460 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "48e20487653845ea968ae05270a44097" instance_seqno: 1790539277425272) as {username='kudu', principal='kudu/127.15.168.2@KRBTEST.COM'} at 127.15.168.2:48211; Asking this server to re-register.
20:01:41.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:40.999734 16658 heartbeater.cc:461] Registering TS with master...
20:01:41.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:41.008293 17460 ts_manager.cc:194] Registered new tserver with Master: 48e20487653845ea968ae05270a44097 (127.15.168.2:39021)
20:01:41.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:41.386090 16806 heartbeater.cc:344] Connected to a master server at 127.15.168.60:45671
20:01:41.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:41.388947 17460 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "83fbdd41d37846738f806b86554e3f41" instance_seqno: 1790539280611250) as {username='kudu', principal='kudu/127.15.168.3@KRBTEST.COM'} at 127.15.168.3:38259; Asking this server to re-register.
20:01:41.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:41.390518 16806 heartbeater.cc:461] Registering TS with master...
20:01:41.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:41.393464 17460 ts_manager.cc:194] Registered new tserver with Master: 83fbdd41d37846738f806b86554e3f41 (127.15.168.3:35707)
20:01:41.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:41.416721 17412 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "4b00615a9e9244d2a54e01564420e235" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } } }
20:01:41.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:41.417487 17412 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [sys.catalog]: This master's current role is: LEADER
20:01:41.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:41.542222 16509 heartbeater.cc:344] Connected to a master server at 127.15.168.60:45671
20:01:41.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:41.545395 17460 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "cd0cbe577a5e4bce9d77779ab1e537e4" instance_seqno: 1790539274711308) as {username='kudu', principal='kudu/127.15.168.1@KRBTEST.COM'} at 127.15.168.1:52845; Asking this server to re-register.
20:01:41.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:41.546958 16509 heartbeater.cc:461] Registering TS with master...
20:01:41.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:41.551131 17460 ts_manager.cc:194] Registered new tserver with Master: cd0cbe577a5e4bce9d77779ab1e537e4 (127.15.168.1:36977)
20:01:41.868 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
20:01:41.869 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
20:01:41.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:41 dist-test-slave-1555 krb5kdc[16044](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539301, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:41.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:01:41.909 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:41.909 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/krb5cc
20:01:41.911 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:01:41.911 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.15.168.61:32863
20:01:41.911 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
20:01:41.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:41.913046 16032 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_muOGM/build/tsan/bin/kudu with pid 17178
20:01:41.950 [DEBUG - kudu-nio-3] (Connection.java:239) [peer master-127.15.168.61:32863(127.15.168.61:32863)] handling channelInactive
20:01:41.951 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.15.168.61:32863(127.15.168.61:32863)] cleaning up while in state READY due to: connection closed
20:01:41.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:41.950903 16548 connection.cc:573] client connection to 127.15.168.61:32863 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
20:01:41.953 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:41.953 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.15.168.62:38531
20:01:41.953 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
20:01:41.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:41.954156 16032 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_muOGM/build/tsan/bin/kudu with pid 17260
20:01:41.985 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:41.986 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.15.168.60:45671
20:01:41.986 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
20:01:41.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:41.986646 16032 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_muOGM/build/tsan/bin/kudu with pid 17372
20:01:42.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:42.019348 16547 connection.cc:573] client connection to 127.15.168.60:45671 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
20:01:42.019 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:42.020 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.15.168.61:32863
20:01:42.020 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
20:01:42.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:42.021750 16032 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_muOGM/build/tsan/bin/kudu
20:01:42.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_muOGM/build/tsan/bin/kudu
20:01:42.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/wal
20:01:42.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/data
20:01:42.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:01:42.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:01:42.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:01:42.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:42.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:42.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:42.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/logs
20:01:42.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/data/info.pb
20:01:42.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:42.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:01:42.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:01:42.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:01:42.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:01:42.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:42.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:42.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:42.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:01:42.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:01:42.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:42.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:42.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.168.61:32863
20:01:42.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.168.61
20:01:42.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35057
20:01:42.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab
20:01:42.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.168.61
20:01:42.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:42.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:01:42.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:01:42.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:01:42.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.168.20:32967
20:01:42.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:42.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:42.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:42.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:42.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.168.62:38531,127.15.168.61:32863,127.15.168.60:45671
20:01:42.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:42.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41913/jwks.json
20:01:42.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:01:42.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:01:42.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:42.347411 17532 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:01:42.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:42.348169 17532 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:01:42.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:42.349244 17532 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:01:42.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:42.384315 17532 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:01:42.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:42.384610 17532 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:01:42.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:42.385063 17532 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:01:42.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:42.385377 17532 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:01:42.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:42.385708 17532 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:01:42.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:42.386102 17532 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:01:42.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:42.386359 17532 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:41913/jwks.json
20:01:42.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:42.428949 17532 master_runner.cc:387] Master server non-default flags:
20:01:42.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/logs
20:01:42.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:42.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:01:42.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:42.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.168.20:32967
20:01:42.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:42.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:42.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/data
20:01:42.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/wal
20:01:42.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:01:42.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:42.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.168.62:38531,127.15.168.61:32863,127.15.168.60:45671
20:01:42.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab
20:01:42.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.168.61
20:01:42.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:42.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:42.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:42.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.168.61:32863
20:01:42.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:42.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:01:42.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:42.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41913/jwks.json
20:01:42.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:42.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:01:42.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:01:42.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:42.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:42.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/data/info.pb
20:01:42.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.168.61
20:01:42.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35057
20:01:42.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:01:42.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:42.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:01:42.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:01:42.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:42.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:01:42.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:01:42.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:01:42.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3b18f0139b1d21a072b7353269301fba16dca54e
20:01:42.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:01:42.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Sep 2026 19:43:26 UTC on e7f111948823
20:01:42.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14030
20:01:42.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:01:42.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:42.430617 17532 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:01:42.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:42.431883 17532 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:01:42.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:42.446187 17540 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:43.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:42.446350 17541 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:43.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:43.796167 17532 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.352s user 0.519s sys 0.814s
20:01:43.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:43.796584 17532 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.352s user 0.519s sys 0.815s
20:01:43.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:43.798262 17532 server_base.cc:1061] running on GCE node
20:01:43.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:43.799989 17549 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:43.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:43.801577 17532 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:01:43.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:43.804817 17532 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:01:43.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:43.806314 17532 hybrid_clock.cc:648] HybridClock initialized: now 1790539303806258 us; error 36 us; skew 500 ppm
20:01:43.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:43.807545 17532 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:01:43.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:43 dist-test-slave-1555 krb5kdc[16044](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539303, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:43.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:43.817464 17532 init.cc:377] Logged in from keytab as kudu/127.15.168.61@KRBTEST.COM (short username kudu)
20:01:43.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:43.824692 17532 webserver.cc:533] Webserver started at http://127.15.168.61:35057/ using document root <none> and password file <none>
20:01:43.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:43.826015 17532 fs_manager.cc:362] Metadata directory not provided
20:01:43.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:43.826320 17532 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:01:43.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:43.838568 17532 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.004s sys 0.004s
20:01:43.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:43.844583 17555 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:01:43.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:43.845844 17532 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
20:01:43.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:43.846136 17532 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/data,/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/wal
20:01:43.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4b00615a9e9244d2a54e01564420e235"
20:01:43.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:01:06 on dist-test-slave-1555"
20:01:43.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:43.848233 17532 fs_report.cc:389] FS layout report
20:01:43.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:01:43.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/wal
20:01:43.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/wal
20:01:43.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/data/data
20:01:43.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:01:43.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:01:43.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:01:43.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:01:43.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:01:43.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:01:43.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:01:43.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:01:43.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:01:43.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:01:43.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:01:43.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:43.934667 17532 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:01:43.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:43.940310 17532 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:01:43.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:43.940831 17532 kserver.cc:163] Server-wide thread pool size limit: 3276
20:01:44.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:44.019433 17532 rpc_server.cc:307] RPC server started. Bound to: 127.15.168.61:32863
20:01:44.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:44.019517 17607 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.168.61:32863 every 8 connection(s)
20:01:44.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:44.022431 17532 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-1/data/info.pb
20:01:44.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:44.030563 16032 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_muOGM/build/tsan/bin/kudu as pid 17532
20:01:44.031 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:44.031 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.15.168.62:38531
20:01:44.031 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
20:01:44.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:44.033509 16032 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_muOGM/build/tsan/bin/kudu
20:01:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_muOGM/build/tsan/bin/kudu
20:01:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/wal
20:01:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/data
20:01:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:01:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:01:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:01:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/logs
20:01:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/data/info.pb
20:01:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:01:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:01:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:01:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:01:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:01:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:01:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.168.62:38531
20:01:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.168.62
20:01:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41983
20:01:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab
20:01:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.168.62
20:01:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:01:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:01:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:01:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.168.20:32967
20:01:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.168.62:38531,127.15.168.61:32863,127.15.168.60:45671
20:01:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41913/jwks.json
20:01:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:01:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:01:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:44.035388 17608 sys_catalog.cc:263] Verifying existing consensus state
20:01:44.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:44.042728 17608 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235: Bootstrap starting.
20:01:44.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:44.082595 17608 log.cc:826] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235: Log is configured to *not* fsync() on all Append() calls
20:01:44.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:44.105079 17608 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:01:44.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:44.106094 17608 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235: Bootstrap complete.
20:01:44.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:44.146235 17608 raft_consensus.cc:359] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } }
20:01:44.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:44.150619 17608 raft_consensus.cc:740] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4b00615a9e9244d2a54e01564420e235, State: Initialized, Role: FOLLOWER
20:01:44.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:44.151918 17608 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } }
20:01:44.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:44.155042 17612 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } } }
20:01:44.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:44.155675 17612 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [sys.catalog]: This master's current role is: FOLLOWER
20:01:44.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:44.157464 17608 sys_catalog.cc:565] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [sys.catalog]: configured and running, proceeding with master startup.
20:01:44.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:44.161067 15536 debug-util.cc:398] Leaking SignalData structure 0x7b080007f780 after lost signal to thread 15467
20:01:44.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:44.161778 15536 debug-util.cc:398] Leaking SignalData structure 0x7b080007fd00 after lost signal to thread 15539
20:01:44.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:44.219475 17623 catalog_manager.cc:1308] Loaded cluster ID: d6c21a38473c4d4189831994cb48f2ce
20:01:44.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:44.219779 17623 catalog_manager.cc:1601] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235: loading cluster ID for follower catalog manager: success
20:01:44.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:44.227581 17623 catalog_manager.cc:1623] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235: acquiring CA information for follower catalog manager: success
20:01:44.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:44.234287 17623 catalog_manager.cc:1651] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:01:44.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:44.207477 17532 master_runner.cc:187] Error getting master registration for 127.15.168.62:38531: Network error: Client connection negotiation failed: client connection to 127.15.168.62:38531: connect: Connection refused (error 111),
20:01:44.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:44.236938 17532 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:01:44.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:44.302147 16807 heartbeater.cc:344] Connected to a master server at 127.15.168.61:32863
20:01:44.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:44.309343 17573 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "83fbdd41d37846738f806b86554e3f41" instance_seqno: 1790539280611250) as {username='kudu', principal='kudu/127.15.168.3@KRBTEST.COM'} at 127.15.168.3:51719; Asking this server to re-register.
20:01:44.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:44.311039 16807 heartbeater.cc:461] Registering TS with master...
20:01:44.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:44.314527 17573 ts_manager.cc:194] Registered new tserver with Master: 83fbdd41d37846738f806b86554e3f41 (127.15.168.3:35707)
20:01:44.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:44.357738 16510 heartbeater.cc:344] Connected to a master server at 127.15.168.61:32863
20:01:44.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:44.360642 17573 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "cd0cbe577a5e4bce9d77779ab1e537e4" instance_seqno: 1790539274711308) as {username='kudu', principal='kudu/127.15.168.1@KRBTEST.COM'} at 127.15.168.1:53079; Asking this server to re-register.
20:01:44.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:44.362058 16510 heartbeater.cc:461] Registering TS with master...
20:01:44.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:44.365226 17573 ts_manager.cc:194] Registered new tserver with Master: cd0cbe577a5e4bce9d77779ab1e537e4 (127.15.168.1:36977)
20:01:44.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:44 dist-test-slave-1555 krb5kdc[16044](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539304, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.1@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:44.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:44.416917 16390 init.cc:295] Successfully reacquired a new kerberos TGT
20:01:44.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:44.442502 17610 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:01:44.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:44.443188 17610 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:01:44.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:44.444183 17610 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:01:44.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:44.497963 17610 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:01:44.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:44.498308 17610 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:01:44.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:44.498662 17610 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:01:44.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:44.498878 17610 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:01:44.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:44.499079 17610 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:01:44.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:44.499274 17610 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:01:44.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:44.499428 17610 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:41913/jwks.json
20:01:44.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:44.540689 17610 master_runner.cc:387] Master server non-default flags:
20:01:44.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/logs
20:01:44.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:44.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:01:44.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:44.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.168.20:32967
20:01:44.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:44.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:44.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/data
20:01:44.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/wal
20:01:44.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:01:44.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:44.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.168.62:38531,127.15.168.61:32863,127.15.168.60:45671
20:01:44.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab
20:01:44.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.168.62
20:01:44.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:44.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:44.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:44.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.168.62:38531
20:01:44.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:44.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:01:44.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:44.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41913/jwks.json
20:01:44.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:44.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:01:44.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:01:44.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:44.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:44.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/data/info.pb
20:01:44.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.168.62
20:01:44.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41983
20:01:44.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:01:44.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:44.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:01:44.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:01:44.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:44.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:01:44.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:01:44.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:01:44.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3b18f0139b1d21a072b7353269301fba16dca54e
20:01:44.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:01:44.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Sep 2026 19:43:26 UTC on e7f111948823
20:01:44.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14030
20:01:44.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:01:44.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:44.541957 17610 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:01:44.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:44.542910 17610 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:01:44.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:44.554556 17635 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:45.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:45.041054 16659 heartbeater.cc:344] Connected to a master server at 127.15.168.61:32863
20:01:45.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:45.044531 17573 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "48e20487653845ea968ae05270a44097" instance_seqno: 1790539277425272) as {username='kudu', principal='kudu/127.15.168.2@KRBTEST.COM'} at 127.15.168.2:57351; Asking this server to re-register.
20:01:45.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:45.046330 16659 heartbeater.cc:461] Registering TS with master...
20:01:45.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:45.050660 17573 ts_manager.cc:194] Registered new tserver with Master: 48e20487653845ea968ae05270a44097 (127.15.168.2:39021)
20:01:45.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:45.465164 17647 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:01:45.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:45.465969 17647 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } }
20:01:45.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:45.470218 17647 leader_election.cc:290] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 95585e2dbd5a475f9466ce816f5fffff (127.15.168.62:38531), 33ab2c6f07d14efa88f25702047f032c (127.15.168.60:45671)
20:01:45.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:45.537515 17558 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.168.60:45671: connect: Connection refused (error 111)
20:01:45.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:45.562215 17559 leader_election.cc:336] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 95585e2dbd5a475f9466ce816f5fffff (127.15.168.62:38531): Network error: Client connection negotiation failed: client connection to 127.15.168.62:38531: connect: Connection refused (error 111)
20:01:45.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:45.565405 17558 leader_election.cc:336] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 33ab2c6f07d14efa88f25702047f032c (127.15.168.60:45671): Network error: Client connection negotiation failed: client connection to 127.15.168.60:45671: connect: Connection refused (error 111)
20:01:45.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:45.566066 17558 leader_election.cc:304] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4b00615a9e9244d2a54e01564420e235; no voters: 33ab2c6f07d14efa88f25702047f032c, 95585e2dbd5a475f9466ce816f5fffff
20:01:45.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:45.567268 17647 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
20:01:47.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:47 dist-test-slave-1555 krb5kdc[16044](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539307, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.2@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:47.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:47.212313 16540 init.cc:295] Successfully reacquired a new kerberos TGT
20:01:47.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:47.384867 17660 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:01:47.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:47.388167 17660 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } }
20:01:47.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:47.409291 17660 leader_election.cc:290] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 95585e2dbd5a475f9466ce816f5fffff (127.15.168.62:38531), 33ab2c6f07d14efa88f25702047f032c (127.15.168.60:45671)
20:01:47.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:47.466732 17558 leader_election.cc:336] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 33ab2c6f07d14efa88f25702047f032c (127.15.168.60:45671): Network error: Client connection negotiation failed: client connection to 127.15.168.60:45671: connect: Connection refused (error 111)
20:01:47.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:47.467991 17559 leader_election.cc:336] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 95585e2dbd5a475f9466ce816f5fffff (127.15.168.62:38531): Network error: Client connection negotiation failed: client connection to 127.15.168.62:38531: connect: Connection refused (error 111)
20:01:47.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:47.468473 17559 leader_election.cc:304] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4b00615a9e9244d2a54e01564420e235; no voters: 33ab2c6f07d14efa88f25702047f032c, 95585e2dbd5a475f9466ce816f5fffff
20:01:47.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:47.476300 17660 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
20:01:47.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:44.556459 17636 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:47.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:45.958926 17634 debug-util.cc:398] Leaking SignalData structure 0x7b0800037cc0 after lost signal to thread 17610
20:01:47.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:47.610953 17610 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 3.055s user 1.013s sys 1.891s
20:01:47.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:47.612677 17610 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 3.057s user 1.013s sys 1.891s
20:01:47.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:47.613632 17638 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:47.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:47.618181 17637 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 3054 milliseconds
20:01:47.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:47.618218 17610 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:01:47.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:47.619653 17610 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:01:47.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:47.622632 17610 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:01:47.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:47.624032 17610 hybrid_clock.cc:648] HybridClock initialized: now 1790539307623973 us; error 65 us; skew 500 ppm
20:01:47.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:47.624985 17610 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:01:47.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:47 dist-test-slave-1555 krb5kdc[16044](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539307, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:47.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:47.631951 17610 init.cc:377] Logged in from keytab as kudu/127.15.168.62@KRBTEST.COM (short username kudu)
20:01:47.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:47.638355 17610 webserver.cc:533] Webserver started at http://127.15.168.62:41983/ using document root <none> and password file <none>
20:01:47.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:47.639542 17610 fs_manager.cc:362] Metadata directory not provided
20:01:47.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:47.639739 17610 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:01:47.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:47.648540 17610 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.002s sys 0.006s
20:01:47.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:47.653894 17671 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:01:47.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:47.655370 17610 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.002s
20:01:47.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:47.655668 17610 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/data,/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/wal
20:01:47.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "95585e2dbd5a475f9466ce816f5fffff"
20:01:47.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:01:04 on dist-test-slave-1555"
20:01:47.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:47.657620 17610 fs_report.cc:389] FS layout report
20:01:47.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:01:47.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/wal
20:01:47.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/wal
20:01:47.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/data/data
20:01:47.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:01:47.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:01:47.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:01:47.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:01:47.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:01:47.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:01:47.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:01:47.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:01:47.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:01:47.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:01:47.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:01:47.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:47.723454 17610 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:01:47.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:47.729523 17610 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:01:47.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:47.729944 17610 kserver.cc:163] Server-wide thread pool size limit: 3276
20:01:47.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:47.820588 17610 rpc_server.cc:307] RPC server started. Bound to: 127.15.168.62:38531
20:01:47.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:47.820639 17722 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.168.62:38531 every 8 connection(s)
20:01:47.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:47.824626 17610 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-0/data/info.pb
20:01:47.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:47.831048 16032 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_muOGM/build/tsan/bin/kudu as pid 17610
20:01:47.831 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:47.832 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.15.168.60:45671
20:01:47.832 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
20:01:47.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:47.834044 16032 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_muOGM/build/tsan/bin/kudu
20:01:47.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_muOGM/build/tsan/bin/kudu
20:01:47.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/wal
20:01:47.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/data
20:01:47.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:01:47.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:01:47.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:01:47.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:47.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:47.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:47.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/logs
20:01:47.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/data/info.pb
20:01:47.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:47.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:01:47.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:01:47.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:01:47.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:01:47.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:47.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:47.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:47.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:01:47.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:01:47.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:47.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:47.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.168.60:45671
20:01:47.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.168.60
20:01:47.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33645
20:01:47.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab
20:01:47.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.168.60
20:01:47.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:47.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:01:47.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:01:47.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:01:47.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.168.20:32967
20:01:47.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:47.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:47.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:47.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:47.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.168.62:38531,127.15.168.61:32863,127.15.168.60:45671
20:01:47.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:47.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41913/jwks.json
20:01:47.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:01:47.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:01:47.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:47.838680 17723 sys_catalog.cc:263] Verifying existing consensus state
20:01:47.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:47.847321 17723 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff: Bootstrap starting.
20:01:47.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:47.890185 17723 log.cc:826] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff: Log is configured to *not* fsync() on all Append() calls
20:01:47.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:47.911567 17723 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:01:47.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:47.912496 17723 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff: Bootstrap complete.
20:01:47.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:47.941933 17723 raft_consensus.cc:359] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } }
20:01:47.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:47.945256 17723 raft_consensus.cc:740] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 95585e2dbd5a475f9466ce816f5fffff, State: Initialized, Role: FOLLOWER
20:01:47.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:47.946458 17723 consensus_queue.cc:260] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } }
20:01:47.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:47.949776 17727 sys_catalog.cc:455] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } } }
20:01:47.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:47.950896 17727 sys_catalog.cc:458] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [sys.catalog]: This master's current role is: FOLLOWER
20:01:47.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:47.951627 17723 sys_catalog.cc:565] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [sys.catalog]: configured and running, proceeding with master startup.
20:01:48.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:48.011165 17738 catalog_manager.cc:1308] Loaded cluster ID: d6c21a38473c4d4189831994cb48f2ce
20:01:48.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:48.011472 17738 catalog_manager.cc:1601] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff: loading cluster ID for follower catalog manager: success
20:01:48.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:48.020341 17738 catalog_manager.cc:1623] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff: acquiring CA information for follower catalog manager: success
20:01:48.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:48.025790 17738 catalog_manager.cc:1651] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:01:48.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:48 dist-test-slave-1555 krb5kdc[16044](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539307, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.62@KRBTEST.COM for kudu/127.15.168.62@KRBTEST.COM
20:01:48.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:48.100322 16660 heartbeater.cc:344] Connected to a master server at 127.15.168.62:38531
20:01:48.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:48.108260 17688 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "48e20487653845ea968ae05270a44097" instance_seqno: 1790539277425272) as {username='kudu', principal='kudu/127.15.168.2@KRBTEST.COM'} at 127.15.168.2:51715; Asking this server to re-register.
20:01:48.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:48.110011 16660 heartbeater.cc:461] Registering TS with master...
20:01:48.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:48.117054 17688 ts_manager.cc:194] Registered new tserver with Master: 48e20487653845ea968ae05270a44097 (127.15.168.2:39021)
20:01:48.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:48.138862 17610 master_runner.cc:187] Error getting master registration for 127.15.168.60:45671: Network error: Client connection negotiation failed: client connection to 127.15.168.60:45671: connect: Connection refused (error 111),
20:01:48.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:48.167161 17610 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:01:48.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:48.362776 17725 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:01:48.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:48.363652 17725 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:01:48.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:48.364817 17725 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:01:48.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:48.408228 16812 heartbeater.cc:344] Connected to a master server at 127.15.168.62:38531
20:01:48.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:48.410792 17688 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "83fbdd41d37846738f806b86554e3f41" instance_seqno: 1790539280611250) as {username='kudu', principal='kudu/127.15.168.3@KRBTEST.COM'} at 127.15.168.3:41481; Asking this server to re-register.
20:01:48.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:48.411962 16812 heartbeater.cc:461] Registering TS with master...
20:01:48.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:48.414638 17688 ts_manager.cc:194] Registered new tserver with Master: 83fbdd41d37846738f806b86554e3f41 (127.15.168.3:35707)
20:01:48.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:48.428583 17725 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:01:48.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:48.428913 17725 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:01:48.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:48.429560 17725 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:01:48.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:48.429962 17725 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:01:48.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:48.430315 17725 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:01:48.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:48.430676 17725 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:01:48.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:48.430938 17725 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:41913/jwks.json
20:01:48.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:48.494925 16511 heartbeater.cc:344] Connected to a master server at 127.15.168.62:38531
20:01:48.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:48.497717 17688 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "cd0cbe577a5e4bce9d77779ab1e537e4" instance_seqno: 1790539274711308) as {username='kudu', principal='kudu/127.15.168.1@KRBTEST.COM'} at 127.15.168.1:37483; Asking this server to re-register.
20:01:48.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:48.499073 16511 heartbeater.cc:461] Registering TS with master...
20:01:48.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:48.501952 17688 ts_manager.cc:194] Registered new tserver with Master: cd0cbe577a5e4bce9d77779ab1e537e4 (127.15.168.1:36977)
20:01:48.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:48.503427 17725 master_runner.cc:387] Master server non-default flags:
20:01:48.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/logs
20:01:48.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:48.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:01:48.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:48.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.168.20:32967
20:01:48.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:48.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:48.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/data
20:01:48.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/wal
20:01:48.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:01:48.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:48.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.168.62:38531,127.15.168.61:32863,127.15.168.60:45671
20:01:48.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/krb5kdc/kudu.keytab
20:01:48.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.168.60
20:01:48.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:48.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:48.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:48.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.168.60:45671
20:01:48.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:48.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:01:48.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:48.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41913/jwks.json
20:01:48.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:48.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:01:48.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:01:48.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:48.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:48.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/data/info.pb
20:01:48.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.168.60
20:01:48.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33645
20:01:48.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:01:48.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:48.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:01:48.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:01:48.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:01:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:01:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:01:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3b18f0139b1d21a072b7353269301fba16dca54e
20:01:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:01:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Sep 2026 19:43:26 UTC on e7f111948823
20:01:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14030
20:01:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:01:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:48.505193 17725 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:01:48.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:48.506300 17725 file_cache.cc:504] Constructed file cache file cache with capacity 419430
20:01:48.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:48.520109 17755 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:49.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:49.343417 17532 master_runner.cc:187] Error getting master registration for 127.15.168.60:45671: Network error: Client connection negotiation failed: client connection to 127.15.168.60:45671: connect: Connection refused (error 111),
20:01:49.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:49.343884 17532 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:01:49.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:49.578572 17766 raft_consensus.cc:493] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:01:49.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:49.579198 17766 raft_consensus.cc:515] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } }
20:01:49.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:49.594692 17583 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "95585e2dbd5a475f9466ce816f5fffff" candidate_term: 4 candidate_status { last_received { term: 3 index: 6 } } ignore_live_leader: false dest_uuid: "4b00615a9e9244d2a54e01564420e235" is_pre_election: true
20:01:49.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:49.595517 17583 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 95585e2dbd5a475f9466ce816f5fffff in term 3.
20:01:49.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:49.597779 17675 leader_election.cc:304] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4b00615a9e9244d2a54e01564420e235, 95585e2dbd5a475f9466ce816f5fffff; no voters:
20:01:49.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:49.601465 17766 leader_election.cc:290] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 4b00615a9e9244d2a54e01564420e235 (127.15.168.61:32863), 33ab2c6f07d14efa88f25702047f032c (127.15.168.60:45671)
20:01:49.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:49.602219 17766 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [term 3 FOLLOWER]: Leader pre-election won for term 4
20:01:49.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:49.602692 17766 raft_consensus.cc:493] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [term 3 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:01:49.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:49.603686 17766 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [term 3 FOLLOWER]: Advancing to term 4
20:01:49.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:49.609611 17674 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.168.60:45671: connect: Connection refused (error 111)
20:01:49.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:49.626320 17674 leader_election.cc:336] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 33ab2c6f07d14efa88f25702047f032c (127.15.168.60:45671): Network error: Client connection negotiation failed: client connection to 127.15.168.60:45671: connect: Connection refused (error 111)
20:01:49.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:49.633921 17766 raft_consensus.cc:515] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } }
20:01:49.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:49.637302 17583 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "95585e2dbd5a475f9466ce816f5fffff" candidate_term: 4 candidate_status { last_received { term: 3 index: 6 } } ignore_live_leader: false dest_uuid: "4b00615a9e9244d2a54e01564420e235"
20:01:49.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:49.637871 17583 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [term 3 FOLLOWER]: Advancing to term 4
20:01:49.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:49.649410 17766 leader_election.cc:290] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [CANDIDATE]: Term 4 election: Requested vote from peers 4b00615a9e9244d2a54e01564420e235 (127.15.168.61:32863), 33ab2c6f07d14efa88f25702047f032c (127.15.168.60:45671)
20:01:49.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:49.648100 17583 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 95585e2dbd5a475f9466ce816f5fffff in term 4.
20:01:49.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:49.655035 17674 leader_election.cc:336] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 33ab2c6f07d14efa88f25702047f032c (127.15.168.60:45671): Network error: Client connection negotiation failed: client connection to 127.15.168.60:45671: connect: Connection refused (error 111)
20:01:49.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:49.655622 17675 leader_election.cc:304] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4b00615a9e9244d2a54e01564420e235, 95585e2dbd5a475f9466ce816f5fffff; no voters: 33ab2c6f07d14efa88f25702047f032c
20:01:49.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:49.657546 17766 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [term 4 FOLLOWER]: Leader election won for term 4
20:01:49.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:49.658157 17766 raft_consensus.cc:697] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [term 4 LEADER]: Becoming Leader. State: Replica: 95585e2dbd5a475f9466ce816f5fffff, State: Running, Role: LEADER
20:01:49.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:49.664991 17766 consensus_queue.cc:237] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } }
20:01:49.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:49.703248 17770 sys_catalog.cc:455] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [sys.catalog]: SysCatalogTable state changed. Reason: New leader 95585e2dbd5a475f9466ce816f5fffff. Latest consensus state: current_term: 4 leader_uuid: "95585e2dbd5a475f9466ce816f5fffff" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } } }
20:01:49.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:49.704531 17770 sys_catalog.cc:458] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [sys.catalog]: This master's current role is: LEADER
20:01:49.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:49.711120 17773 catalog_manager.cc:1524] Loading table and tablet metadata into memory...
20:01:49.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:49.741542 17773 catalog_manager.cc:1533] Initializing Kudu cluster ID...
20:01:49.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:49.753039 17773 catalog_manager.cc:1308] Loaded cluster ID: d6c21a38473c4d4189831994cb48f2ce
20:01:49.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:49.755888 17773 catalog_manager.cc:1544] Initializing Kudu internal certificate authority...
20:01:49.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:49.798898 17773 catalog_manager.cc:1553] Loading token signing keys...
20:01:49.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:49.846159 17773 catalog_manager.cc:6230] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff: Loaded TSK: 0
20:01:49.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:49.852633 17773 catalog_manager.cc:1563] Initializing in-progress tserver states...
20:01:50.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:50 dist-test-slave-1555 krb5kdc[16044](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539310, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.3@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:50.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:50.086153 16689 init.cc:295] Successfully reacquired a new kerberos TGT
20:01:50.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:50.203436 17674 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.15.168.60:45671, user_credentials={real_user=slave}} blocked reactor thread for 48210.6us
20:01:50.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:50.210364 17674 consensus_peers.cc:597] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff -> Peer 33ab2c6f07d14efa88f25702047f032c (127.15.168.60:45671): Couldn't send request to peer 33ab2c6f07d14efa88f25702047f032c. Status: Network error: Client connection negotiation failed: client connection to 127.15.168.60:45671: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:01:50.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:50.273612 17583 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [term 4 FOLLOWER]: Refusing update from remote peer 95585e2dbd5a475f9466ce816f5fffff: Log matching property violated. Preceding OpId in replica: term: 3 index: 6. Preceding OpId from leader: term: 4 index: 7. (index mismatch)
20:01:50.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:50.278568 17770 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [LEADER]: Connected to new peer: Peer: permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.001s
20:01:50.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:50.340924 17776 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 95585e2dbd5a475f9466ce816f5fffff. Latest consensus state: current_term: 4 leader_uuid: "95585e2dbd5a475f9466ce816f5fffff" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } } }
20:01:50.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:50.349370 17776 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [sys.catalog]: This master's current role is: FOLLOWER
20:01:50.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:50.356536 17770 sys_catalog.cc:455] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "95585e2dbd5a475f9466ce816f5fffff" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } } }
20:01:50.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:50.357262 17770 sys_catalog.cc:458] T 00000000000000000000000000000000 P 95585e2dbd5a475f9466ce816f5fffff [sys.catalog]: This master's current role is: LEADER
20:01:50.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:50.365847 17776 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "95585e2dbd5a475f9466ce816f5fffff" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } } }
20:01:50.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:50.368914 17776 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4b00615a9e9244d2a54e01564420e235 [sys.catalog]: This master's current role is: FOLLOWER
20:01:51.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:49.923265 17753 debug-util.cc:398] Leaking SignalData structure 0x7b0800037cc0 after lost signal to thread 17725
20:01:51.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:51.022076 17753 kernel_stack_watchdog.cc:198] Thread 17725 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 401ms:
20:01:51.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:01:51.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:01:51.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:01:51.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:01:51.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:48.520687 17754 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:51.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:51.024313 17757 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:51.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:51.024961 17756 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2502 milliseconds
20:01:51.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:51.028761 17725 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.509s user 0.647s sys 1.479s
20:01:51.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:51.029129 17725 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.510s user 0.648s sys 1.479s
20:01:51.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:51.029410 17725 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:01:51.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:51.030859 17725 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:01:51.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:51.034157 17725 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:01:51.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:51.035624 17725 hybrid_clock.cc:648] HybridClock initialized: now 1790539311035581 us; error 50 us; skew 500 ppm
20:01:51.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:51.036775 17725 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:01:51.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:01:51 dist-test-slave-1555 krb5kdc[16044](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539311, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.168.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:51.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:51.045921 17725 init.cc:377] Logged in from keytab as kudu/127.15.168.60@KRBTEST.COM (short username kudu)
20:01:51.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:51.052294 17725 webserver.cc:533] Webserver started at http://127.15.168.60:33645/ using document root <none> and password file <none>
20:01:51.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:51.053774 17725 fs_manager.cc:362] Metadata directory not provided
20:01:51.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:51.054235 17725 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:01:51.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:51.066546 17725 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.006s sys 0.003s
20:01:51.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:51.073187 17789 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:01:51.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:51.074667 17725 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.001s sys 0.002s
20:01:51.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:51.075075 17725 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/data,/tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/wal
20:01:51.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "33ab2c6f07d14efa88f25702047f032c"
20:01:51.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:01:08 on dist-test-slave-1555"
20:01:51.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:51.077870 17725 fs_report.cc:389] FS layout report
20:01:51.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:01:51.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/wal
20:01:51.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/wal
20:01:51.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/data/data
20:01:51.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:01:51.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:01:51.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:01:51.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:01:51.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:01:51.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:01:51.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:01:51.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:01:51.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:01:51.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:01:51.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:01:51.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:51.131201 16660 heartbeater.cc:499] Master 127.15.168.62:38531 was elected leader, sending a full tablet report...
20:01:51.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:51.153898 17725 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:01:51.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:51.165926 17725 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:01:51.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:51.166518 17725 kserver.cc:163] Server-wide thread pool size limit: 3276
20:01:51.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:51.268899 17725 rpc_server.cc:307] RPC server started. Bound to: 127.15.168.60:45671
20:01:51.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:51.268931 17841 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.168.60:45671 every 8 connection(s)
20:01:51.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:51.272964 17725 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_muOGM/test-tmp/mini-kudu-cluster17477250656521094859/master-2/data/info.pb
20:01:51.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:51.281909 16032 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_muOGM/build/tsan/bin/kudu as pid 17725
20:01:51.283 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:51.283 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.15.168.61:32863(127.15.168.61:32863)] connecting to peer
20:01:51.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:51.285724 17842 sys_catalog.cc:263] Verifying existing consensus state
20:01:51.287 [DEBUG - kudu-nio-4] (Connection.java:757) [peer master-127.15.168.61:32863(127.15.168.61:32863)] Successfully connected to peer
20:01:51.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:51.294795 17842 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c: Bootstrap starting.
20:01:51.306 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.15.168.61:32863(127.15.168.61:32863)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
20:01:51.309 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.15.168.62:38531(127.15.168.62:38531)] disconnecting while in state TERMINATED
20:01:51.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:01:51.310156 17845 negotiation.cc:336] Failed RPC negotiation. Trace:
20:01:51.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 20:01:51.286441 (+ 0us) reactor.cc:730] Submitting negotiation task for server connection from 127.0.0.1:40610 (local address 127.15.168.61:32863)
20:01:51.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 20:01:51.287866 (+ 1425us) server_negotiation.cc:207] Beginning negotiation
20:01:51.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 20:01:51.287896 (+ 30us) server_negotiation.cc:400] Waiting for connection header
20:01:51.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 20:01:51.297647 (+ 9751us) server_negotiation.cc:408] Connection header received
20:01:51.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 20:01:51.299052 (+ 1405us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
20:01:51.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 20:01:51.299085 (+ 33us) server_negotiation.cc:462] Received NEGOTIATE request from client
20:01:51.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 20:01:51.299401 (+ 316us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
20:01:51.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 20:01:51.299666 (+ 265us) server_negotiation.cc:227] Negotiated authn=SASL
20:01:51.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 20:01:51.308571 (+ 8905us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:40610: BlockingRecv error: recv got EOF from 127.0.0.1:40610 (error 108)
20:01:51.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":855,"thread_start_us":562,"threads_started":1}
20:01:51.312 [DEBUG - kudu-nio-4] (Connection.java:239) [peer master-127.15.168.61:32863(127.15.168.61:32863)] handling channelInactive
20:01:51.312 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.15.168.60:45671(127.15.168.60:45671)] disconnecting while in state TERMINATED
20:01:51.312 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.15.168.61:32863(127.15.168.61:32863)] disconnecting while in state TERMINATED
20:01:51.314 [DEBUG - Time-limited test] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:01:51.321 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:01:51.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:51.324116 16032 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_muOGM/build/tsan/bin/kudu with pid 16375
20:01:51.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:51.354550 16032 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_muOGM/build/tsan/bin/kudu with pid 16523
20:01:51.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:51.358551 17842 log.cc:826] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c: Log is configured to *not* fsync() on all Append() calls
20:01:51.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:51.386230 17842 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:01:51.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:51.388119 16032 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_muOGM/build/tsan/bin/kudu with pid 16672
20:01:51.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:51.387261 17842 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c: Bootstrap complete.
20:01:51.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:51.419283 16032 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_muOGM/build/tsan/bin/kudu with pid 17610
20:01:51.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:51.429495 17842 raft_consensus.cc:359] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } }
20:01:51.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:51.432217 17842 raft_consensus.cc:740] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 33ab2c6f07d14efa88f25702047f032c, State: Initialized, Role: FOLLOWER
20:01:51.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:51.433329 17842 consensus_queue.cc:260] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } }
20:01:51.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:51.436103 17853 sys_catalog.cc:455] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95585e2dbd5a475f9466ce816f5fffff" member_type: VOTER last_known_addr { host: "127.15.168.62" port: 38531 } } peers { permanent_uuid: "4b00615a9e9244d2a54e01564420e235" member_type: VOTER last_known_addr { host: "127.15.168.61" port: 32863 } } peers { permanent_uuid: "33ab2c6f07d14efa88f25702047f032c" member_type: VOTER last_known_addr { host: "127.15.168.60" port: 45671 } } }
20:01:51.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:51.437042 17853 sys_catalog.cc:458] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [sys.catalog]: This master's current role is: FOLLOWER
20:01:51.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:51.438104 17842 sys_catalog.cc:565] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c [sys.catalog]: configured and running, proceeding with master startup.
20:01:51.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:51.462469 16032 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_muOGM/build/tsan/bin/kudu with pid 17532
20:01:51.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:51.477099 17864 catalog_manager.cc:1308] Loaded cluster ID: d6c21a38473c4d4189831994cb48f2ce
20:01:51.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:51.477406 17864 catalog_manager.cc:1601] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c: loading cluster ID for follower catalog manager: success
20:01:51.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:51.485342 17864 catalog_manager.cc:1623] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c: acquiring CA information for follower catalog manager: success
20:01:51.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:51.491035 17864 catalog_manager.cc:1651] T 00000000000000000000000000000000 P 33ab2c6f07d14efa88f25702047f032c: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:01:51.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:51.489230 17725 master_runner.cc:187] Error getting master registration for 127.15.168.62:38531: Network error: Client connection negotiation failed: client connection to 127.15.168.62:38531: connect: Connection refused (error 111),
20:01:51.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:51.519236 16032 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_muOGM/build/tsan/bin/kudu with pid 17725
20:01:52.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T20:01:52Z chronyd exiting
20:01:53.781 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:01:53.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:53.784255 15305 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_muOGM/build/tsan/bin/kudu with pid 15584
20:01:53.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:53.811975 15305 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_muOGM/build/tsan/bin/kudu with pid 15742
20:01:53.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:53.841586 15305 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_muOGM/build/tsan/bin/kudu with pid 15889
20:01:53.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:53.872311 15305 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_muOGM/build/tsan/bin/kudu with pid 15317
20:01:53.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:53.904368 15305 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_muOGM/build/tsan/bin/kudu with pid 15387
20:01:53.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:01:53.935856 15305 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_muOGM/build/tsan/bin/kudu with pid 15462
20:01:55.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T20:01:55Z chronyd exiting
20:01:56.139 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:160)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:456)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:564)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:432)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:366)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:93)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:160)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:456)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:564)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:432)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:366)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:93)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
20:01:57.459 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
20:01:57.459 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:160)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:456)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:564)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:432)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:366)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:93)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:160)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:456)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:564)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:432)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:366)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:93)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more