03:07:21.247 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
03:07:21.259 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskuwp0L7/test-tmp
03:07:21.263 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
03:07:21.266 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskuwp0L7/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
03:07:21.509 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
03:07:21.702 [DEBUG - main] (MiniKuduCluster.java:195) Response:
03:07:21.704 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
03:07:21.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:21.707621 13750 mini_oidc.cc:150] Starting JWKS server
03:07:21.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:21.711485 13750 webserver.cc:492] Webserver started at http://127.0.0.1:41251/ using document root <none> and password file <none>
03:07:21.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:21.711926 13750 mini_oidc.cc:186] Starting OIDC Discovery server
03:07:21.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:21.713706 13750 webserver.cc:492] Webserver started at http://127.0.0.1:36753/ using document root <none> and password file <none>
03:07:21.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-01-29T03:07:21Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
03:07:21.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-01-29T03:07:21Z Disabled control of system clock
03:07:21.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:21.774346 13750 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu
03:07:21.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu
03:07:21.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-0/wal
03:07:21.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-0/data
03:07:21.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:21.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:21.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:21.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:21.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:21.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:21.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-0/logs
03:07:21.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-0/data/info.pb
03:07:21.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:21.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:21.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:21.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:21.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:21.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:21.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:21.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:21.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:07:21.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:21.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:21.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:21.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.109.190:38537
03:07:21.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.109.190
03:07:21.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:21.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.109.148:36253
03:07:21.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:21.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:21.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:21.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:21.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.109.190:38537,127.13.109.189:33927,127.13.109.188:33151
03:07:21.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:21.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41251/jwks.json with env {}
03:07:22.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:22.082195 13762 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:22.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:22.082863 13762 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:22.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:22.083447 13762 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:22.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:22.115648 13762 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:07:22.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:22.115983 13762 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:22.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:22.116180 13762 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:07:22.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:22.116411 13762 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:07:22.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:22.116691 13762 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:07:22.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:22.116848 13762 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41251/jwks.json
03:07:22.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:22.153502 13762 master_runner.cc:386] Master server non-default flags:
03:07:22.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:22.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.109.148:36253
03:07:22.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:22.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:22.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-0/data
03:07:22.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-0/wal
03:07:22.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:22.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.109.190:38537,127.13.109.189:33927,127.13.109.188:33151
03:07:22.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:22.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:22.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:22.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.109.190:38537
03:07:22.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:22.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:22.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:22.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41251/jwks.json
03:07:22.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:22.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:22.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-0/data/info.pb
03:07:22.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.109.190
03:07:22.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:22.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:22.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:22.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:22.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:22.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:22.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-0/logs
03:07:22.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:22.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:22.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:22.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:07:22.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:07:22.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
03:07:22.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:22.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Jan 2026 02:43:21 UTC on 5fd53c4cbb9d
03:07:22.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10241
03:07:22.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:22.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:22.155036 13762 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:22.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:22.156908 13762 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:22.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:22.169664 13768 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:23.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:22.169902 13769 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:23.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:23.362980 13771 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:23.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:23.365444 13770 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1191 milliseconds
03:07:23.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:23.365572 13762 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:07:23.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:23.367100 13762 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:23.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:23.370361 13762 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:23.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:23.371800 13762 hybrid_clock.cc:648] HybridClock initialized: now 1769656043371754 us; error 57 us; skew 500 ppm
03:07:23.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:23.372705 13762 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:23.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:23.379503 13762 webserver.cc:492] Webserver started at http://127.13.109.190:41701/ using document root <none> and password file <none>
03:07:23.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:23.380525 13762 fs_manager.cc:362] Metadata directory not provided
03:07:23.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:23.380730 13762 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:23.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:23.381139 13762 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
03:07:23.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:23.385480 13762 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-0/data/instance:
03:07:23.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "480f1c6bb9614e098ae66cb1528f154c"
03:07:23.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-29 03:07:23 on dist-test-slave-7fb5"
03:07:23.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:23.386677 13762 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-0/wal/instance:
03:07:23.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "480f1c6bb9614e098ae66cb1528f154c"
03:07:23.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-29 03:07:23 on dist-test-slave-7fb5"
03:07:23.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:23.393944 13762 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.003s
03:07:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:23.399948 13778 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:23.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:23.401281 13762 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
03:07:23.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:23.401683 13762 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-0/data,/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-0/wal
03:07:23.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "480f1c6bb9614e098ae66cb1528f154c"
03:07:23.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-29 03:07:23 on dist-test-slave-7fb5"
03:07:23.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:23.402118 13762 fs_report.cc:389] FS layout report
03:07:23.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:23.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-0/wal
03:07:23.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-0/wal
03:07:23.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-0/data/data
03:07:23.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:23.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:23.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:23.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:23.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:23.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:23.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:23.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:23.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:23.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:23.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:23.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:23.456259 13762 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:23.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:23.457962 13762 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:23.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:23.458511 13762 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:23.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:23.532670 13762 rpc_server.cc:307] RPC server started. Bound to: 127.13.109.190:38537
03:07:23.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:23.532732 13830 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.109.190:38537 every 8 connection(s)
03:07:23.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:23.535583 13762 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-0/data/info.pb
03:07:23.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:23.539944 13750 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu as pid 13762
03:07:23.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:23.540508 13750 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-0/wal/instance
03:07:23.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:23.543093 13831 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:07:23.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:23.545619 13750 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu
03:07:23.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu
03:07:23.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-1/wal
03:07:23.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-1/data
03:07:23.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:23.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:23.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:23.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:23.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:23.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:23.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-1/logs
03:07:23.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-1/data/info.pb
03:07:23.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:23.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:23.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:23.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:23.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:23.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:23.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:23.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:23.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:07:23.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:23.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:23.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:23.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.109.189:33927
03:07:23.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.109.189
03:07:23.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:23.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.109.148:36253
03:07:23.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:23.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:23.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:23.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:23.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.109.190:38537,127.13.109.189:33927,127.13.109.188:33151
03:07:23.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:23.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41251/jwks.json with env {}
03:07:23.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:23.556648 13831 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.109.190" port: 38537 } has no permanent_uuid. Determining permanent_uuid...
03:07:23.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:23.584873 13831 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.109.189" port: 33927 } has no permanent_uuid. Determining permanent_uuid...
03:07:23.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:23.587594 13780 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.13.109.189:33927: connect: Connection refused (error 111)
03:07:23.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:23.592074 13831 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.13.109.189:33927: Network error: Client connection negotiation failed: client connection to 127.13.109.189:33927: connect: Connection refused (error 111)
03:07:23.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:23.641598 13831 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.109.189" port: 33927 } attempt: 1
03:07:23.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:23.645587 13831 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.13.109.189:33927: Network error: Client connection negotiation failed: client connection to 127.13.109.189:33927: connect: Connection refused (error 111)
03:07:23.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:23.714041 13831 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.109.189" port: 33927 } attempt: 2
03:07:23.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:23.717901 13831 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.13.109.189:33927: Network error: Client connection negotiation failed: client connection to 127.13.109.189:33927: connect: Connection refused (error 111)
03:07:23.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:23.809341 13831 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.109.189" port: 33927 } attempt: 3
03:07:23.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:23.813829 13831 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.13.109.189:33927: Network error: Client connection negotiation failed: client connection to 127.13.109.189:33927: connect: Connection refused (error 111)
03:07:23.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:23.856557 13833 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:23.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:23.857115 13833 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:23.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:23.857636 13833 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:23.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:23.889906 13833 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:07:23.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:23.890199 13833 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:23.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:23.890465 13833 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:07:23.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:23.890661 13833 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:07:23.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:23.890851 13833 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:07:23.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:23.891003 13833 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41251/jwks.json
03:07:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:23.927970 13833 master_runner.cc:386] Master server non-default flags:
03:07:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.109.148:36253
03:07:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-1/data
03:07:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-1/wal
03:07:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.109.190:38537,127.13.109.189:33927,127.13.109.188:33151
03:07:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.109.189:33927
03:07:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41251/jwks.json
03:07:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-1/data/info.pb
03:07:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.109.189
03:07:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-1/logs
03:07:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:07:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:07:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
03:07:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:23.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Jan 2026 02:43:21 UTC on 5fd53c4cbb9d
03:07:23.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10241
03:07:23.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:23.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:23.929225 13833 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:23.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:23.931347 13833 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:23.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:23.942930 13842 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:23.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:23.957275 13831 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.109.189" port: 33927 } attempt: 4
03:07:23.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:23.962646 13831 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.13.109.189:33927: Network error: Client connection negotiation failed: client connection to 127.13.109.189:33927: connect: Connection refused (error 111)
03:07:24.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:24.262048 13831 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.109.189" port: 33927 } attempt: 5
03:07:24.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:24.266754 13831 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.13.109.189:33927: Network error: Client connection negotiation failed: client connection to 127.13.109.189:33927: connect: Connection refused (error 111)
03:07:24.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:24.814184 13831 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.109.189" port: 33927 } attempt: 6
03:07:24.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:24.822003 13831 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.13.109.189:33927: Network error: Client connection negotiation failed: client connection to 127.13.109.189:33927: connect: Connection refused (error 111)
03:07:25.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:23.944375 13843 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:25.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:25.133287 13844 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
03:07:25.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:25.135149 13845 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:25.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:25.135283 13833 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:07:25.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:25.140595 13833 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:25.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:25.143920 13833 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:25.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:25.145360 13833 hybrid_clock.cc:648] HybridClock initialized: now 1769656045145326 us; error 45 us; skew 500 ppm
03:07:25.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:25.146430 13833 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:25.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:25.153887 13833 webserver.cc:492] Webserver started at http://127.13.109.189:35627/ using document root <none> and password file <none>
03:07:25.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:25.155117 13833 fs_manager.cc:362] Metadata directory not provided
03:07:25.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:25.155400 13833 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:25.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:25.156081 13833 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
03:07:25.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:25.160683 13833 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-1/data/instance:
03:07:25.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2a12b94c3a85402999751a6922535714"
03:07:25.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-29 03:07:25 on dist-test-slave-7fb5"
03:07:25.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:25.162148 13833 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-1/wal/instance:
03:07:25.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2a12b94c3a85402999751a6922535714"
03:07:25.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-29 03:07:25 on dist-test-slave-7fb5"
03:07:25.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:25.169610 13833 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.000s sys 0.008s
03:07:25.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:25.175710 13854 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:25.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:25.176929 13833 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
03:07:25.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:25.177297 13833 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-1/data,/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-1/wal
03:07:25.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2a12b94c3a85402999751a6922535714"
03:07:25.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-29 03:07:25 on dist-test-slave-7fb5"
03:07:25.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:25.177696 13833 fs_report.cc:389] FS layout report
03:07:25.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:25.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-1/wal
03:07:25.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-1/wal
03:07:25.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-1/data/data
03:07:25.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:25.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:25.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:25.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:25.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:25.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:25.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:25.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:25.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:25.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:25.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:25.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:25.242671 13833 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:25.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:25.244489 13833 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:25.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:25.245064 13833 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:25.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:25.316416 13833 rpc_server.cc:307] RPC server started. Bound to: 127.13.109.189:33927
03:07:25.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:25.316551 13905 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.109.189:33927 every 8 connection(s)
03:07:25.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:25.319247 13833 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-1/data/info.pb
03:07:25.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:25.324360 13750 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu as pid 13833
03:07:25.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:25.324812 13750 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-1/wal/instance
03:07:25.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:25.325388 13906 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:07:25.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:25.327026 13750 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu
03:07:25.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu
03:07:25.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-2/wal
03:07:25.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-2/data
03:07:25.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:25.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:25.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:25.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:25.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:25.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:25.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-2/logs
03:07:25.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-2/data/info.pb
03:07:25.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:25.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:25.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:25.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:25.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:25.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:25.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:25.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:25.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:07:25.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:25.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:25.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:25.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.109.188:33151
03:07:25.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.109.188
03:07:25.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:25.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.109.148:36253
03:07:25.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:25.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:25.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:25.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:25.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.109.190:38537,127.13.109.189:33927,127.13.109.188:33151
03:07:25.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:25.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41251/jwks.json with env {}
03:07:25.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:25.339543 13906 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.109.190" port: 38537 } has no permanent_uuid. Determining permanent_uuid...
03:07:25.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:25.366158 13906 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.109.189" port: 33927 } has no permanent_uuid. Determining permanent_uuid...
03:07:25.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:25.380851 13906 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.109.188" port: 33151 } has no permanent_uuid. Determining permanent_uuid...
03:07:25.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:25.383715 13855 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.13.109.188:33151: connect: Connection refused (error 111)
03:07:25.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:25.388401 13906 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.13.109.188:33151: Network error: Client connection negotiation failed: client connection to 127.13.109.188:33151: connect: Connection refused (error 111)
03:07:25.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:25.437958 13906 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.109.188" port: 33151 } attempt: 1
03:07:25.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:25.443130 13906 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.13.109.188:33151: Network error: Client connection negotiation failed: client connection to 127.13.109.188:33151: connect: Connection refused (error 111)
03:07:25.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:25.511606 13906 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.109.188" port: 33151 } attempt: 2
03:07:25.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:25.516407 13906 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.13.109.188:33151: Network error: Client connection negotiation failed: client connection to 127.13.109.188:33151: connect: Connection refused (error 111)
03:07:25.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:25.607923 13906 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.109.188" port: 33151 } attempt: 3
03:07:25.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:25.612200 13906 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.13.109.188:33151: Network error: Client connection negotiation failed: client connection to 127.13.109.188:33151: connect: Connection refused (error 111)
03:07:25.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:25.660261 13908 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:25.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:25.660948 13908 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:25.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:25.661638 13908 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:25.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:25.693312 13908 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:07:25.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:25.693678 13908 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:25.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:25.694010 13908 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:07:25.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:25.694391 13908 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:07:25.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:25.694725 13908 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:07:25.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:25.694976 13908 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41251/jwks.json
03:07:25.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:25.730824 13908 master_runner.cc:386] Master server non-default flags:
03:07:25.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:25.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.109.148:36253
03:07:25.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:25.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:25.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-2/data
03:07:25.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-2/wal
03:07:25.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:25.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.109.190:38537,127.13.109.189:33927,127.13.109.188:33151
03:07:25.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:25.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:25.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:25.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.109.188:33151
03:07:25.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:25.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:25.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:25.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41251/jwks.json
03:07:25.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:25.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:25.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-2/data/info.pb
03:07:25.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.109.188
03:07:25.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:25.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:25.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:25.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:25.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:25.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:25.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-2/logs
03:07:25.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:25.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:25.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:25.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:07:25.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:07:25.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
03:07:25.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:25.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Jan 2026 02:43:21 UTC on 5fd53c4cbb9d
03:07:25.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10241
03:07:25.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:25.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:25.732582 13908 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:25.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:25.734884 13908 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:25.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:25.747555 13918 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:25.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:25.755599 13906 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.109.188" port: 33151 } attempt: 4
03:07:25.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:25.761334 13906 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.13.109.188:33151: Network error: Client connection negotiation failed: client connection to 127.13.109.188:33151: connect: Connection refused (error 111)
03:07:25.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:25.882431 13831 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.109.189" port: 33927 } attempt: 7
03:07:25.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:25.910673 13831 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.109.188" port: 33151 } has no permanent_uuid. Determining permanent_uuid...
03:07:25.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:25.919857 13831 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.13.109.188:33151: Network error: Client connection negotiation failed: client connection to 127.13.109.188:33151: connect: Connection refused (error 111)
03:07:25.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:25.978562 13831 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.109.188" port: 33151 } attempt: 1
03:07:25.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:25.985519 13831 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.13.109.188:33151: Network error: Client connection negotiation failed: client connection to 127.13.109.188:33151: connect: Connection refused (error 111)
03:07:26.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:26.060900 13906 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.109.188" port: 33151 } attempt: 5
03:07:26.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:26.067123 13831 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.109.188" port: 33151 } attempt: 2
03:07:26.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:26.067967 13906 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.13.109.188:33151: Network error: Client connection negotiation failed: client connection to 127.13.109.188:33151: connect: Connection refused (error 111)
03:07:26.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:26.073967 13831 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.13.109.188:33151: Network error: Client connection negotiation failed: client connection to 127.13.109.188:33151: connect: Connection refused (error 111)
03:07:26.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:26.159577 13831 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.109.188" port: 33151 } attempt: 3
03:07:26.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:26.166114 13831 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.13.109.188:33151: Network error: Client connection negotiation failed: client connection to 127.13.109.188:33151: connect: Connection refused (error 111)
03:07:26.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:26.306695 13831 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.109.188" port: 33151 } attempt: 4
03:07:26.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:26.313202 13831 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.13.109.188:33151: Network error: Client connection negotiation failed: client connection to 127.13.109.188:33151: connect: Connection refused (error 111)
03:07:26.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:26.596764 13831 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.109.188" port: 33151 } attempt: 5
03:07:26.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:26.603365 13831 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.13.109.188:33151: Network error: Client connection negotiation failed: client connection to 127.13.109.188:33151: connect: Connection refused (error 111)
03:07:26.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:26.615563 13906 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.109.188" port: 33151 } attempt: 6
03:07:26.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:26.627379 13906 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.13.109.188:33151: Network error: Client connection negotiation failed: client connection to 127.13.109.188:33151: connect: Connection refused (error 111)
03:07:26.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:25.749008 13919 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:26.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:26.993883 13921 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:26.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:26.995699 13920 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
03:07:26.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:26.996582 13908 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.250s user 0.457s sys 0.791s
03:07:26.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:26.996981 13908 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.250s user 0.457s sys 0.791s
03:07:26.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:26.997274 13908 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:07:26.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:26.998891 13908 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:27.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.002578 13908 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:27.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.004330 13908 hybrid_clock.cc:648] HybridClock initialized: now 1769656047004252 us; error 87 us; skew 500 ppm
03:07:27.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.005589 13908 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:27.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.015499 13908 webserver.cc:492] Webserver started at http://127.13.109.188:34479/ using document root <none> and password file <none>
03:07:27.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.017092 13908 fs_manager.cc:362] Metadata directory not provided
03:07:27.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.017433 13908 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:27.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.018262 13908 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
03:07:27.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.025885 13908 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-2/data/instance:
03:07:27.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6d5be82a2cf1433da98fd6c797340193"
03:07:27.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-29 03:07:27 on dist-test-slave-7fb5"
03:07:27.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.027627 13908 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-2/wal/instance:
03:07:27.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6d5be82a2cf1433da98fd6c797340193"
03:07:27.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-29 03:07:27 on dist-test-slave-7fb5"
03:07:27.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.038211 13908 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.006s sys 0.002s
03:07:27.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.046437 13933 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:27.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.047653 13908 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.006s sys 0.000s
03:07:27.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.048032 13908 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-2/data,/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-2/wal
03:07:27.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6d5be82a2cf1433da98fd6c797340193"
03:07:27.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-29 03:07:27 on dist-test-slave-7fb5"
03:07:27.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.048357 13908 fs_report.cc:389] FS layout report
03:07:27.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:27.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-2/wal
03:07:27.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-2/wal
03:07:27.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-2/data/data
03:07:27.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:27.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:27.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:27.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:27.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:27.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:27.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:27.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:27.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:27.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:27.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:27.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.120893 13908 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:27.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.122344 13908 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:27.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.122745 13908 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:27.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.155966 13831 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.109.188" port: 33151 } attempt: 6
03:07:27.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:27.163936 13831 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.13.109.188:33151: Network error: Client connection negotiation failed: client connection to 127.13.109.188:33151: connect: Connection refused (error 111)
03:07:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.195679 13908 rpc_server.cc:307] RPC server started. Bound to: 127.13.109.188:33151
03:07:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.195772 13986 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.109.188:33151 every 8 connection(s)
03:07:27.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.198231 13908 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-2/data/info.pb
03:07:27.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.202709 13750 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu as pid 13908
03:07:27.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.203193 13750 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/master-2/wal/instance
03:07:27.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.206091 13987 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:07:27.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.233160 13987 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.109.190" port: 38537 } has no permanent_uuid. Determining permanent_uuid...
03:07:27.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.257541 13987 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.109.189" port: 33927 } has no permanent_uuid. Determining permanent_uuid...
03:07:27.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.267408 13987 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.109.188" port: 33151 } has no permanent_uuid. Determining permanent_uuid...
03:07:27.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.282689 13987 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6d5be82a2cf1433da98fd6c797340193: Bootstrap starting.
03:07:27.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.288928 13987 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 6d5be82a2cf1433da98fd6c797340193: Neither blocks nor log segments found. Creating new log.
03:07:27.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.291251 13987 log.cc:826] T 00000000000000000000000000000000 P 6d5be82a2cf1433da98fd6c797340193: Log is configured to *not* fsync() on all Append() calls
03:07:27.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.295742 13987 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6d5be82a2cf1433da98fd6c797340193: No bootstrap required, opened a new log
03:07:27.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.320535 13987 raft_consensus.cc:359] T 00000000000000000000000000000000 P 6d5be82a2cf1433da98fd6c797340193 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "480f1c6bb9614e098ae66cb1528f154c" member_type: VOTER last_known_addr { host: "127.13.109.190" port: 38537 } } peers { permanent_uuid: "2a12b94c3a85402999751a6922535714" member_type: VOTER last_known_addr { host: "127.13.109.189" port: 33927 } } peers { permanent_uuid: "6d5be82a2cf1433da98fd6c797340193" member_type: VOTER last_known_addr { host: "127.13.109.188" port: 33151 } }
03:07:27.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.321553 13987 raft_consensus.cc:385] T 00000000000000000000000000000000 P 6d5be82a2cf1433da98fd6c797340193 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:07:27.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.321890 13987 raft_consensus.cc:740] T 00000000000000000000000000000000 P 6d5be82a2cf1433da98fd6c797340193 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d5be82a2cf1433da98fd6c797340193, State: Initialized, Role: FOLLOWER
03:07:27.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.322935 13987 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6d5be82a2cf1433da98fd6c797340193 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "480f1c6bb9614e098ae66cb1528f154c" member_type: VOTER last_known_addr { host: "127.13.109.190" port: 38537 } } peers { permanent_uuid: "2a12b94c3a85402999751a6922535714" member_type: VOTER last_known_addr { host: "127.13.109.189" port: 33927 } } peers { permanent_uuid: "6d5be82a2cf1433da98fd6c797340193" member_type: VOTER last_known_addr { host: "127.13.109.188" port: 33151 } }
03:07:27.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.325758 13996 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6d5be82a2cf1433da98fd6c797340193 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "480f1c6bb9614e098ae66cb1528f154c" member_type: VOTER last_known_addr { host: "127.13.109.190" port: 38537 } } peers { permanent_uuid: "2a12b94c3a85402999751a6922535714" member_type: VOTER last_known_addr { host: "127.13.109.189" port: 33927 } } peers { permanent_uuid: "6d5be82a2cf1433da98fd6c797340193" member_type: VOTER last_known_addr { host: "127.13.109.188" port: 33151 } } }
03:07:27.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.326387 13996 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6d5be82a2cf1433da98fd6c797340193 [sys.catalog]: This master's current role is: FOLLOWER
03:07:27.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.327932 13987 sys_catalog.cc:565] T 00000000000000000000000000000000 P 6d5be82a2cf1433da98fd6c797340193 [sys.catalog]: configured and running, proceeding with master startup.
03:07:27.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:27.358462 14007 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 6d5be82a2cf1433da98fd6c797340193: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:07:27.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:27.358831 14007 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:07:27.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:27.378454 13939 tablet.cc:2378] T 00000000000000000000000000000000 P 6d5be82a2cf1433da98fd6c797340193: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:07:27.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.353169 13908 master_runner.cc:186] Error getting master registration for 127.13.109.190:38537: OK, instance_id { permanent_uuid: "480f1c6bb9614e098ae66cb1528f154c" instance_seqno: 1769656043511236 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:07:27.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.382790 13908 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:07:27.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.575320 13996 raft_consensus.cc:493] T 00000000000000000000000000000000 P 6d5be82a2cf1433da98fd6c797340193 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:07:27.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.575731 13996 raft_consensus.cc:515] T 00000000000000000000000000000000 P 6d5be82a2cf1433da98fd6c797340193 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "480f1c6bb9614e098ae66cb1528f154c" member_type: VOTER last_known_addr { host: "127.13.109.190" port: 38537 } } peers { permanent_uuid: "2a12b94c3a85402999751a6922535714" member_type: VOTER last_known_addr { host: "127.13.109.189" port: 33927 } } peers { permanent_uuid: "6d5be82a2cf1433da98fd6c797340193" member_type: VOTER last_known_addr { host: "127.13.109.188" port: 33151 } }
03:07:27.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.577740 13996 leader_election.cc:290] T 00000000000000000000000000000000 P 6d5be82a2cf1433da98fd6c797340193 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 480f1c6bb9614e098ae66cb1528f154c (127.13.109.190:38537), 2a12b94c3a85402999751a6922535714 (127.13.109.189:33927)
03:07:27.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.687958 13906 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.109.188" port: 33151 } attempt: 7
03:07:27.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.715062 13906 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2a12b94c3a85402999751a6922535714: Bootstrap starting.
03:07:27.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.722735 13906 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 2a12b94c3a85402999751a6922535714: Neither blocks nor log segments found. Creating new log.
03:07:27.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.725158 13906 log.cc:826] T 00000000000000000000000000000000 P 2a12b94c3a85402999751a6922535714: Log is configured to *not* fsync() on all Append() calls
03:07:27.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.730548 13906 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2a12b94c3a85402999751a6922535714: No bootstrap required, opened a new log
03:07:27.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.759207 13906 raft_consensus.cc:359] T 00000000000000000000000000000000 P 2a12b94c3a85402999751a6922535714 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "480f1c6bb9614e098ae66cb1528f154c" member_type: VOTER last_known_addr { host: "127.13.109.190" port: 38537 } } peers { permanent_uuid: "2a12b94c3a85402999751a6922535714" member_type: VOTER last_known_addr { host: "127.13.109.189" port: 33927 } } peers { permanent_uuid: "6d5be82a2cf1433da98fd6c797340193" member_type: VOTER last_known_addr { host: "127.13.109.188" port: 33151 } }
03:07:27.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.760177 13906 raft_consensus.cc:385] T 00000000000000000000000000000000 P 2a12b94c3a85402999751a6922535714 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:07:27.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.760517 13906 raft_consensus.cc:740] T 00000000000000000000000000000000 P 2a12b94c3a85402999751a6922535714 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2a12b94c3a85402999751a6922535714, State: Initialized, Role: FOLLOWER
03:07:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.761469 13906 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2a12b94c3a85402999751a6922535714 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "480f1c6bb9614e098ae66cb1528f154c" member_type: VOTER last_known_addr { host: "127.13.109.190" port: 38537 } } peers { permanent_uuid: "2a12b94c3a85402999751a6922535714" member_type: VOTER last_known_addr { host: "127.13.109.189" port: 33927 } } peers { permanent_uuid: "6d5be82a2cf1433da98fd6c797340193" member_type: VOTER last_known_addr { host: "127.13.109.188" port: 33151 } }
03:07:27.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.767545 13906 sys_catalog.cc:565] T 00000000000000000000000000000000 P 2a12b94c3a85402999751a6922535714 [sys.catalog]: configured and running, proceeding with master startup.
03:07:27.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.767745 13881 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6d5be82a2cf1433da98fd6c797340193" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2a12b94c3a85402999751a6922535714" is_pre_election: true
03:07:27.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.768610 13881 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 2a12b94c3a85402999751a6922535714 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d5be82a2cf1433da98fd6c797340193 in term 0.
03:07:27.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.766139 14011 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2a12b94c3a85402999751a6922535714 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "480f1c6bb9614e098ae66cb1528f154c" member_type: VOTER last_known_addr { host: "127.13.109.190" port: 38537 } } peers { permanent_uuid: "2a12b94c3a85402999751a6922535714" member_type: VOTER last_known_addr { host: "127.13.109.189" port: 33927 } } peers { permanent_uuid: "6d5be82a2cf1433da98fd6c797340193" member_type: VOTER last_known_addr { host: "127.13.109.188" port: 33151 } } }
03:07:27.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.769924 14011 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2a12b94c3a85402999751a6922535714 [sys.catalog]: This master's current role is: FOLLOWER
03:07:27.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.774045 13935 leader_election.cc:304] T 00000000000000000000000000000000 P 6d5be82a2cf1433da98fd6c797340193 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2a12b94c3a85402999751a6922535714, 6d5be82a2cf1433da98fd6c797340193; no voters:
03:07:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.775352 13996 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 6d5be82a2cf1433da98fd6c797340193 [term 0 FOLLOWER]: Leader pre-election won for term 1
03:07:27.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.775790 13996 raft_consensus.cc:493] T 00000000000000000000000000000000 P 6d5be82a2cf1433da98fd6c797340193 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:07:27.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.776201 13996 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 6d5be82a2cf1433da98fd6c797340193 [term 0 FOLLOWER]: Advancing to term 1
03:07:27.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.795995 13996 raft_consensus.cc:515] T 00000000000000000000000000000000 P 6d5be82a2cf1433da98fd6c797340193 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "480f1c6bb9614e098ae66cb1528f154c" member_type: VOTER last_known_addr { host: "127.13.109.190" port: 38537 } } peers { permanent_uuid: "2a12b94c3a85402999751a6922535714" member_type: VOTER last_known_addr { host: "127.13.109.189" port: 33927 } } peers { permanent_uuid: "6d5be82a2cf1433da98fd6c797340193" member_type: VOTER last_known_addr { host: "127.13.109.188" port: 33151 } }
03:07:27.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.803479 13996 leader_election.cc:290] T 00000000000000000000000000000000 P 6d5be82a2cf1433da98fd6c797340193 [CANDIDATE]: Term 1 election: Requested vote from peers 480f1c6bb9614e098ae66cb1528f154c (127.13.109.190:38537), 2a12b94c3a85402999751a6922535714 (127.13.109.189:33927)
03:07:27.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.805493 13881 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6d5be82a2cf1433da98fd6c797340193" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2a12b94c3a85402999751a6922535714"
03:07:27.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.806311 13881 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 2a12b94c3a85402999751a6922535714 [term 0 FOLLOWER]: Advancing to term 1
03:07:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.818665 13881 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 2a12b94c3a85402999751a6922535714 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d5be82a2cf1433da98fd6c797340193 in term 1.
03:07:27.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.819823 13935 leader_election.cc:304] T 00000000000000000000000000000000 P 6d5be82a2cf1433da98fd6c797340193 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2a12b94c3a85402999751a6922535714, 6d5be82a2cf1433da98fd6c797340193; no voters:
03:07:27.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.820809 13996 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 6d5be82a2cf1433da98fd6c797340193 [term 1 FOLLOWER]: Leader election won for term 1
03:07:27.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.834344 13996 raft_consensus.cc:697] T 00000000000000000000000000000000 P 6d5be82a2cf1433da98fd6c797340193 [term 1 LEADER]: Becoming Leader. State: Replica: 6d5be82a2cf1433da98fd6c797340193, State: Running, Role: LEADER
03:07:27.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.835556 13996 consensus_queue.cc:237] T 00000000000000000000000000000000 P 6d5be82a2cf1433da98fd6c797340193 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "480f1c6bb9614e098ae66cb1528f154c" member_type: VOTER last_known_addr { host: "127.13.109.190" port: 38537 } } peers { permanent_uuid: "2a12b94c3a85402999751a6922535714" member_type: VOTER last_known_addr { host: "127.13.109.189" port: 33927 } } peers { permanent_uuid: "6d5be82a2cf1433da98fd6c797340193" member_type: VOTER last_known_addr { host: "127.13.109.188" port: 33151 } }
03:07:27.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.843677 14023 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6d5be82a2cf1433da98fd6c797340193 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6d5be82a2cf1433da98fd6c797340193. Latest consensus state: current_term: 1 leader_uuid: "6d5be82a2cf1433da98fd6c797340193" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "480f1c6bb9614e098ae66cb1528f154c" member_type: VOTER last_known_addr { host: "127.13.109.190" port: 38537 } } peers { permanent_uuid: "2a12b94c3a85402999751a6922535714" member_type: VOTER last_known_addr { host: "127.13.109.189" port: 33927 } } peers { permanent_uuid: "6d5be82a2cf1433da98fd6c797340193" member_type: VOTER last_known_addr { host: "127.13.109.188" port: 33151 } } }
03:07:27.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.844879 14023 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6d5be82a2cf1433da98fd6c797340193 [sys.catalog]: This master's current role is: LEADER
03:07:27.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.847246 14026 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
03:07:27.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.854975 14026 catalog_manager.cc:1494] Initializing Kudu cluster ID...
03:07:27.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:27.873437 14024 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 2a12b94c3a85402999751a6922535714: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:07:27.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:27.873822 14024 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:07:27.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.875276 13881 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 2a12b94c3a85402999751a6922535714 [term 1 FOLLOWER]: Refusing update from remote peer 6d5be82a2cf1433da98fd6c797340193: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:07:27.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.877170 13996 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 6d5be82a2cf1433da98fd6c797340193 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2a12b94c3a85402999751a6922535714" member_type: VOTER last_known_addr { host: "127.13.109.189" port: 33927 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:07:27.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.831135 13833 master_runner.cc:186] Error getting master registration for 127.13.109.190:38537: OK, instance_id { permanent_uuid: "480f1c6bb9614e098ae66cb1528f154c" instance_seqno: 1769656043511236 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:07:27.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.896478 13833 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:07:27.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.915174 14011 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2a12b94c3a85402999751a6922535714 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6d5be82a2cf1433da98fd6c797340193. Latest consensus state: current_term: 1 leader_uuid: "6d5be82a2cf1433da98fd6c797340193" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "480f1c6bb9614e098ae66cb1528f154c" member_type: VOTER last_known_addr { host: "127.13.109.190" port: 38537 } } peers { permanent_uuid: "2a12b94c3a85402999751a6922535714" member_type: VOTER last_known_addr { host: "127.13.109.189" port: 33927 } } peers { permanent_uuid: "6d5be82a2cf1433da98fd6c797340193" member_type: VOTER last_known_addr { host: "127.13.109.188" port: 33151 } } }
03:07:27.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.915923 14011 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2a12b94c3a85402999751a6922535714 [sys.catalog]: This master's current role is: FOLLOWER
03:07:27.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.925366 14027 mvcc.cc:204] Tried to move back new op lower bound from 7248511172067815424 to 7248511171946008576. Current Snapshot: MvccSnapshot[applied={T|T < 7248511172067815424}]
03:07:27.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.932049 14029 mvcc.cc:204] Tried to move back new op lower bound from 7248511172067815424 to 7248511171946008576. Current Snapshot: MvccSnapshot[applied={T|T < 7248511172067815424}]
03:07:27.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.949775 14023 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6d5be82a2cf1433da98fd6c797340193 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "6d5be82a2cf1433da98fd6c797340193" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "480f1c6bb9614e098ae66cb1528f154c" member_type: VOTER last_known_addr { host: "127.13.109.190" port: 38537 } } peers { permanent_uuid: "2a12b94c3a85402999751a6922535714" member_type: VOTER last_known_addr { host: "127.13.109.189" port: 33927 } } peers { permanent_uuid: "6d5be82a2cf1433da98fd6c797340193" member_type: VOTER last_known_addr { host: "127.13.109.188" port: 33151 } } }
03:07:27.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.950554 14023 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6d5be82a2cf1433da98fd6c797340193 [sys.catalog]: This master's current role is: LEADER
03:07:27.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.957140 14011 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2a12b94c3a85402999751a6922535714 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "6d5be82a2cf1433da98fd6c797340193" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "480f1c6bb9614e098ae66cb1528f154c" member_type: VOTER last_known_addr { host: "127.13.109.190" port: 38537 } } peers { permanent_uuid: "2a12b94c3a85402999751a6922535714" member_type: VOTER last_known_addr { host: "127.13.109.189" port: 33927 } } peers { permanent_uuid: "6d5be82a2cf1433da98fd6c797340193" member_type: VOTER last_known_addr { host: "127.13.109.188" port: 33151 } } }
03:07:27.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.957917 14011 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2a12b94c3a85402999751a6922535714 [sys.catalog]: This master's current role is: FOLLOWER
03:07:27.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.964776 14026 catalog_manager.cc:1357] Generated new cluster ID: 778d1d940a004a1caa407a6a5854bfe2
03:07:27.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:27.965103 14026 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
03:07:28.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:28.002206 14026 catalog_manager.cc:1380] Generated new certificate authority record
03:07:28.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:28.005092 14026 catalog_manager.cc:1514] Loading token signing keys...
03:07:28.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:28.054220 14026 catalog_manager.cc:6027] T 00000000000000000000000000000000 P 6d5be82a2cf1433da98fd6c797340193: Generated new TSK 0
03:07:28.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:28.055361 14026 catalog_manager.cc:1524] Initializing in-progress tserver states...
03:07:28.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:28.090740 13750 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu
03:07:28.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu
03:07:28.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-0/wal
03:07:28.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-0/data
03:07:28.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:28.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:28.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:28.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:28.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:28.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:28.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-0/logs
03:07:28.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-0/data/info.pb
03:07:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:07:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.109.129:0
03:07:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.109.129
03:07:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.109.129
03:07:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.109.190:38537,127.13.109.189:33927,127.13.109.188:33151
03:07:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.109.148:36253
03:07:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:28.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
03:07:28.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:28.197326 13831 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.109.188" port: 33151 } attempt: 7
03:07:28.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:28.273809 13831 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 480f1c6bb9614e098ae66cb1528f154c: Bootstrap starting.
03:07:28.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:28.317757 13831 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 480f1c6bb9614e098ae66cb1528f154c: Neither blocks nor log segments found. Creating new log.
03:07:28.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:28.331442 13831 log.cc:826] T 00000000000000000000000000000000 P 480f1c6bb9614e098ae66cb1528f154c: Log is configured to *not* fsync() on all Append() calls
03:07:28.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:28.359999 13831 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 480f1c6bb9614e098ae66cb1528f154c: No bootstrap required, opened a new log
03:07:28.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:28.553649 13831 raft_consensus.cc:359] T 00000000000000000000000000000000 P 480f1c6bb9614e098ae66cb1528f154c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "480f1c6bb9614e098ae66cb1528f154c" member_type: VOTER last_known_addr { host: "127.13.109.190" port: 38537 } } peers { permanent_uuid: "2a12b94c3a85402999751a6922535714" member_type: VOTER last_known_addr { host: "127.13.109.189" port: 33927 } } peers { permanent_uuid: "6d5be82a2cf1433da98fd6c797340193" member_type: VOTER last_known_addr { host: "127.13.109.188" port: 33151 } }
03:07:28.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:28.562379 13831 raft_consensus.cc:385] T 00000000000000000000000000000000 P 480f1c6bb9614e098ae66cb1528f154c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:07:28.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:28.566805 13831 raft_consensus.cc:740] T 00000000000000000000000000000000 P 480f1c6bb9614e098ae66cb1528f154c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 480f1c6bb9614e098ae66cb1528f154c, State: Initialized, Role: FOLLOWER
03:07:28.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:28.576301 13831 consensus_queue.cc:260] T 00000000000000000000000000000000 P 480f1c6bb9614e098ae66cb1528f154c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "480f1c6bb9614e098ae66cb1528f154c" member_type: VOTER last_known_addr { host: "127.13.109.190" port: 38537 } } peers { permanent_uuid: "2a12b94c3a85402999751a6922535714" member_type: VOTER last_known_addr { host: "127.13.109.189" port: 33927 } } peers { permanent_uuid: "6d5be82a2cf1433da98fd6c797340193" member_type: VOTER last_known_addr { host: "127.13.109.188" port: 33151 } }
03:07:28.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:28.597987 14040 sys_catalog.cc:455] T 00000000000000000000000000000000 P 480f1c6bb9614e098ae66cb1528f154c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "480f1c6bb9614e098ae66cb1528f154c" member_type: VOTER last_known_addr { host: "127.13.109.190" port: 38537 } } peers { permanent_uuid: "2a12b94c3a85402999751a6922535714" member_type: VOTER last_known_addr { host: "127.13.109.189" port: 33927 } } peers { permanent_uuid: "6d5be82a2cf1433da98fd6c797340193" member_type: VOTER last_known_addr { host: "127.13.109.188" port: 33151 } } }
03:07:28.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:28.598814 14040 sys_catalog.cc:458] T 00000000000000000000000000000000 P 480f1c6bb9614e098ae66cb1528f154c [sys.catalog]: This master's current role is: FOLLOWER
03:07:28.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:28.602902 14033 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:28.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:28.603495 14033 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:28.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:28.604312 14033 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:28.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:28.610661 13804 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6d5be82a2cf1433da98fd6c797340193" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "480f1c6bb9614e098ae66cb1528f154c"
03:07:28.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:28.611441 13804 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 480f1c6bb9614e098ae66cb1528f154c [term 0 FOLLOWER]: Advancing to term 1
03:07:28.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:28.616565 13805 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6d5be82a2cf1433da98fd6c797340193" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "480f1c6bb9614e098ae66cb1528f154c" is_pre_election: true
03:07:28.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:28.617666 13805 rpcz_store.cc:275] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:46808 (request call id 2) took 1039 ms. Trace:
03:07:28.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:28.618021 13805 rpcz_store.cc:276] 0129 03:07:27.578172 (+ 0us) service_pool.cc:168] Inserting onto call queue
03:07:28.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0129 03:07:27.578338 (+ 166us) service_pool.cc:225] Handling call
03:07:28.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0129 03:07:28.617595 (+1039257us) inbound_call.cc:173] Queueing success response
03:07:28.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
03:07:28.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:28.622573 13804 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 480f1c6bb9614e098ae66cb1528f154c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d5be82a2cf1433da98fd6c797340193 in term 1.
03:07:28.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:28.624058 13803 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 480f1c6bb9614e098ae66cb1528f154c [term 1 FOLLOWER]: Refusing update from remote peer 6d5be82a2cf1433da98fd6c797340193: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:07:28.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:28.630601 13996 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 6d5be82a2cf1433da98fd6c797340193 [LEADER]: Connected to new peer: Peer: permanent_uuid: "480f1c6bb9614e098ae66cb1528f154c" member_type: VOTER last_known_addr { host: "127.13.109.190" port: 38537 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:07:28.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:28.637378 13831 sys_catalog.cc:565] T 00000000000000000000000000000000 P 480f1c6bb9614e098ae66cb1528f154c [sys.catalog]: configured and running, proceeding with master startup.
03:07:28.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:28.667253 14033 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:28.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:28.670231 14033 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.13.109.129
03:07:28.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:28.728752 14040 sys_catalog.cc:455] T 00000000000000000000000000000000 P 480f1c6bb9614e098ae66cb1528f154c [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6d5be82a2cf1433da98fd6c797340193. Latest consensus state: current_term: 1 leader_uuid: "6d5be82a2cf1433da98fd6c797340193" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "480f1c6bb9614e098ae66cb1528f154c" member_type: VOTER last_known_addr { host: "127.13.109.190" port: 38537 } } peers { permanent_uuid: "2a12b94c3a85402999751a6922535714" member_type: VOTER last_known_addr { host: "127.13.109.189" port: 33927 } } peers { permanent_uuid: "6d5be82a2cf1433da98fd6c797340193" member_type: VOTER last_known_addr { host: "127.13.109.188" port: 33151 } } }
03:07:28.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:28.729971 14040 sys_catalog.cc:458] T 00000000000000000000000000000000 P 480f1c6bb9614e098ae66cb1528f154c [sys.catalog]: This master's current role is: FOLLOWER
03:07:28.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:28.729187 14047 sys_catalog.cc:455] T 00000000000000000000000000000000 P 480f1c6bb9614e098ae66cb1528f154c [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "6d5be82a2cf1433da98fd6c797340193" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "480f1c6bb9614e098ae66cb1528f154c" member_type: VOTER last_known_addr { host: "127.13.109.190" port: 38537 } } peers { permanent_uuid: "2a12b94c3a85402999751a6922535714" member_type: VOTER last_known_addr { host: "127.13.109.189" port: 33927 } } peers { permanent_uuid: "6d5be82a2cf1433da98fd6c797340193" member_type: VOTER last_known_addr { host: "127.13.109.188" port: 33151 } } }
03:07:28.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:28.732928 14047 sys_catalog.cc:458] T 00000000000000000000000000000000 P 480f1c6bb9614e098ae66cb1528f154c [sys.catalog]: This master's current role is: FOLLOWER
03:07:28.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:28.734659 14033 tablet_server_runner.cc:78] Tablet server non-default flags:
03:07:28.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:28.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.109.148:36253
03:07:28.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:28.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:28.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-0/data
03:07:28.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-0/wal
03:07:28.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:28.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:28.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.109.129:0
03:07:28.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:28.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:28.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:28.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-0/data/info.pb
03:07:28.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.109.129
03:07:28.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.109.190:38537,127.13.109.189:33927,127.13.109.188:33151
03:07:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.109.129
03:07:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-0/logs
03:07:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:07:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:07:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
03:07:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Jan 2026 02:43:21 UTC on 5fd53c4cbb9d
03:07:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10241
03:07:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:28.736438 14033 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:28.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:28.738898 14033 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:28.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:28.760097 14057 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:28.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:28.780987 14060 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
03:07:28.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:28.880375 14024 catalog_manager.cc:1269] Loaded cluster ID: 778d1d940a004a1caa407a6a5854bfe2
03:07:28.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:28.881114 14024 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 2a12b94c3a85402999751a6922535714: loading cluster ID for follower catalog manager: success
03:07:28.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:28.888475 14024 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 2a12b94c3a85402999751a6922535714: acquiring CA information for follower catalog manager: success
03:07:28.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:28.906714 14024 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 2a12b94c3a85402999751a6922535714: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:07:29.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:29.241091 13996 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6d5be82a2cf1433da98fd6c797340193 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "6d5be82a2cf1433da98fd6c797340193" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "480f1c6bb9614e098ae66cb1528f154c" member_type: VOTER last_known_addr { host: "127.13.109.190" port: 38537 } } peers { permanent_uuid: "2a12b94c3a85402999751a6922535714" member_type: VOTER last_known_addr { host: "127.13.109.189" port: 33927 } } peers { permanent_uuid: "6d5be82a2cf1433da98fd6c797340193" member_type: VOTER last_known_addr { host: "127.13.109.188" port: 33151 } } }
03:07:29.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:29.242761 13996 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6d5be82a2cf1433da98fd6c797340193 [sys.catalog]: This master's current role is: LEADER
03:07:29.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:29.823412 14060 catalog_manager.cc:1269] Loaded cluster ID: 778d1d940a004a1caa407a6a5854bfe2
03:07:29.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:29.823774 14060 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 480f1c6bb9614e098ae66cb1528f154c: loading cluster ID for follower catalog manager: success
03:07:29.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:29.846334 14060 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 480f1c6bb9614e098ae66cb1528f154c: acquiring CA information for follower catalog manager: success
03:07:29.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:29.865747 14060 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 480f1c6bb9614e098ae66cb1528f154c: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:07:30.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:30.162683 14056 debug-util.cc:398] Leaking SignalData structure 0x7b080002ec60 after lost signal to thread 14033
03:07:30.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:30.849903 14056 kernel_stack_watchdog.cc:198] Thread 14033 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 398ms:
03:07:30.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
03:07:30.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
03:07:30.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
03:07:30.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
03:07:30.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:28.763490 14058 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:30.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:30.851267 14033 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.090s user 0.716s sys 1.270s
03:07:30.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:30.851603 14033 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.090s user 0.716s sys 1.270s
03:07:30.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:30.861851 14033 server_base.cc:1060] running on GCE node
03:07:30.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:30.863415 14066 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:30.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:30.864902 14033 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:30.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:30.867928 14033 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:30.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:30.869437 14033 hybrid_clock.cc:648] HybridClock initialized: now 1769656050869242 us; error 203 us; skew 500 ppm
03:07:30.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:30.870347 14033 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:30.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:30.877600 14033 webserver.cc:492] Webserver started at http://127.13.109.129:45037/ using document root <none> and password file <none>
03:07:30.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:30.878656 14033 fs_manager.cc:362] Metadata directory not provided
03:07:30.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:30.878911 14033 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:30.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:30.879473 14033 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
03:07:30.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:30.885707 14033 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-0/data/instance:
03:07:30.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "579610a03c434abdb616a57912ca9715"
03:07:30.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-29 03:07:30 on dist-test-slave-7fb5"
03:07:30.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:30.887207 14033 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-0/wal/instance:
03:07:30.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "579610a03c434abdb616a57912ca9715"
03:07:30.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-29 03:07:30 on dist-test-slave-7fb5"
03:07:30.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:30.896891 14033 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.007s sys 0.000s
03:07:30.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:30.904634 14071 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:30.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:30.906090 14033 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.001s sys 0.003s
03:07:30.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:30.906472 14033 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-0/data,/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-0/wal
03:07:30.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "579610a03c434abdb616a57912ca9715"
03:07:30.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-29 03:07:30 on dist-test-slave-7fb5"
03:07:30.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:30.906914 14033 fs_report.cc:389] FS layout report
03:07:30.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:30.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-0/wal
03:07:30.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-0/wal
03:07:30.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-0/data/data
03:07:30.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:30.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:30.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:30.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:30.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:30.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:30.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:30.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:30.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:30.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:30.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:30.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:30.975324 14033 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:30.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:30.977317 14033 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:30.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:30.977883 14033 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:30.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:30.981186 14033 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
03:07:30.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:30.986898 14033 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:07:30.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:30.987183 14033 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:07:30.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:30.987489 14033 ts_tablet_manager.cc:616] Registered 0 tablets
03:07:30.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:30.987705 14033 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:07:31.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:31.157709 14033 rpc_server.cc:307] RPC server started. Bound to: 127.13.109.129:46861
03:07:31.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:31.157824 14184 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.109.129:46861 every 8 connection(s)
03:07:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:31.160398 14033 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-0/data/info.pb
03:07:31.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:31.161125 13750 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu as pid 14033
03:07:31.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:31.161583 13750 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-0/wal/instance
03:07:31.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:31.197871 13750 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu
03:07:31.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu
03:07:31.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-1/wal
03:07:31.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-1/data
03:07:31.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:31.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:31.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:31.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:31.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:31.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:31.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-1/logs
03:07:31.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-1/data/info.pb
03:07:31.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:31.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:31.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:31.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:31.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:31.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:31.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:31.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:31.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:07:31.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:31.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.109.130:0
03:07:31.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.109.130
03:07:31.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.109.130
03:07:31.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:31.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.109.190:38537,127.13.109.189:33927,127.13.109.188:33151
03:07:31.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.109.148:36253
03:07:31.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:31.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:31.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
03:07:31.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:31.255216 14185 heartbeater.cc:344] Connected to a master server at 127.13.109.188:33151
03:07:31.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:31.264952 14185 heartbeater.cc:461] Registering TS with master...
03:07:31.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:31.266362 14185 heartbeater.cc:507] Master 127.13.109.188:33151 requested a full tablet report, sending...
03:07:31.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:31.259945 14186 heartbeater.cc:344] Connected to a master server at 127.13.109.189:33927
03:07:31.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:31.267971 14186 heartbeater.cc:461] Registering TS with master...
03:07:31.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:31.270403 13950 ts_manager.cc:194] Registered new tserver with Master: 579610a03c434abdb616a57912ca9715 (127.13.109.129:46861)
03:07:31.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:31.273334 13950 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.13.109.129:40351
03:07:31.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:31.279624 14186 heartbeater.cc:507] Master 127.13.109.189:33927 requested a full tablet report, sending...
03:07:31.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:31.283052 13871 ts_manager.cc:194] Registered new tserver with Master: 579610a03c434abdb616a57912ca9715 (127.13.109.129:46861)
03:07:31.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:31.284561 14187 heartbeater.cc:344] Connected to a master server at 127.13.109.190:38537
03:07:31.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:31.284919 14187 heartbeater.cc:461] Registering TS with master...
03:07:31.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:31.285557 14187 heartbeater.cc:507] Master 127.13.109.190:38537 requested a full tablet report, sending...
03:07:31.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:31.288487 13795 ts_manager.cc:194] Registered new tserver with Master: 579610a03c434abdb616a57912ca9715 (127.13.109.129:46861)
03:07:31.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:31.597867 14192 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:31.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:31.598438 14192 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:31.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:31.599086 14192 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:31.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:31.632491 14192 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:31.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:31.633473 14192 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.13.109.130
03:07:31.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:31.672482 14192 tablet_server_runner.cc:78] Tablet server non-default flags:
03:07:31.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:31.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.109.148:36253
03:07:31.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:31.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:31.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-1/data
03:07:31.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-1/wal
03:07:31.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:31.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:31.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.109.130:0
03:07:31.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:31.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:31.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:31.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-1/data/info.pb
03:07:31.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.109.130
03:07:31.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:31.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.109.190:38537,127.13.109.189:33927,127.13.109.188:33151
03:07:31.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:31.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:31.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:31.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:31.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:31.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.109.130
03:07:31.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-1/logs
03:07:31.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:31.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:31.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:31.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:07:31.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:07:31.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
03:07:31.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:31.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Jan 2026 02:43:21 UTC on 5fd53c4cbb9d
03:07:31.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10241
03:07:31.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:31.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:31.674216 14192 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:31.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:31.676062 14192 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:31.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:31.689872 14202 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:32.279204 14185 heartbeater.cc:499] Master 127.13.109.188:33151 was elected leader, sending a full tablet report...
03:07:33.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:33.094703 14201 debug-util.cc:398] Leaking SignalData structure 0x7b080002ec60 after lost signal to thread 14192
03:07:33.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:33.553944 14201 kernel_stack_watchdog.cc:198] Thread 14192 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 399ms:
03:07:33.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
03:07:33.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
03:07:33.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
03:07:33.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
03:07:33.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:31.691171 14203 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:33.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:33.555228 14192 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.863s user 0.001s sys 0.000s
03:07:33.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:33.555585 14192 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.864s user 0.002s sys 0.000s
03:07:33.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:33.557639 14205 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:33.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:33.560379 14204 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1867 milliseconds
03:07:33.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:33.560410 14192 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:07:33.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:33.562055 14192 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:33.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:33.564638 14192 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:33.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:33.566229 14192 hybrid_clock.cc:648] HybridClock initialized: now 1769656053566157 us; error 80 us; skew 500 ppm
03:07:33.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:33.567415 14192 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:33.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:33.576212 14192 webserver.cc:492] Webserver started at http://127.13.109.130:36577/ using document root <none> and password file <none>
03:07:33.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:33.577389 14192 fs_manager.cc:362] Metadata directory not provided
03:07:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:33.577834 14192 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:33.578500 14192 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
03:07:33.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:33.584755 14192 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-1/data/instance:
03:07:33.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6519e447bab048089e2f4a13a83c7d14"
03:07:33.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-29 03:07:33 on dist-test-slave-7fb5"
03:07:33.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:33.586267 14192 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-1/wal/instance:
03:07:33.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6519e447bab048089e2f4a13a83c7d14"
03:07:33.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-29 03:07:33 on dist-test-slave-7fb5"
03:07:33.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:33.594799 14192 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.005s sys 0.004s
03:07:33.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:33.601503 14213 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:33.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:33.602702 14192 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.003s
03:07:33.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:33.603001 14192 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-1/data,/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-1/wal
03:07:33.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6519e447bab048089e2f4a13a83c7d14"
03:07:33.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-29 03:07:33 on dist-test-slave-7fb5"
03:07:33.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:33.603494 14192 fs_report.cc:389] FS layout report
03:07:33.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:33.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-1/wal
03:07:33.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-1/wal
03:07:33.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-1/data/data
03:07:33.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:33.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:33.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:33.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:33.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:33.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:33.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:33.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:33.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:33.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:33.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:33.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:33.655833 14192 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:33.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:33.657380 14192 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:33.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:33.657775 14192 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:33.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:33.660214 14192 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
03:07:33.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:33.664258 14192 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:07:33.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:33.664448 14192 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:07:33.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:33.664633 14192 ts_tablet_manager.cc:616] Registered 0 tablets
03:07:33.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:33.664770 14192 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:07:33.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:33.798162 14192 rpc_server.cc:307] RPC server started. Bound to: 127.13.109.130:43349
03:07:33.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:33.798220 14326 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.109.130:43349 every 8 connection(s)
03:07:33.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:33.800732 14192 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-1/data/info.pb
03:07:33.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:33.810714 13750 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu as pid 14192
03:07:33.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:33.811312 13750 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-1/wal/instance
03:07:33.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:33.828051 13750 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu
03:07:33.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu
03:07:33.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-2/wal
03:07:33.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-2/data
03:07:33.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:33.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:33.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:33.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:33.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:33.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:33.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-2/logs
03:07:33.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-2/data/info.pb
03:07:33.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:33.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:33.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:33.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:33.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:33.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:33.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:33.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:33.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:07:33.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:33.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.109.131:0
03:07:33.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.109.131
03:07:33.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.109.131
03:07:33.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:33.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.109.190:38537,127.13.109.189:33927,127.13.109.188:33151
03:07:33.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.109.148:36253
03:07:33.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:33.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:33.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
03:07:33.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:33.841135 14327 heartbeater.cc:344] Connected to a master server at 127.13.109.188:33151
03:07:33.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:33.841750 14327 heartbeater.cc:461] Registering TS with master...
03:07:33.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:33.843262 14327 heartbeater.cc:507] Master 127.13.109.188:33151 requested a full tablet report, sending...
03:07:33.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:33.847296 13950 ts_manager.cc:194] Registered new tserver with Master: 6519e447bab048089e2f4a13a83c7d14 (127.13.109.130:43349)
03:07:33.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:33.849308 13950 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.13.109.130:55307
03:07:33.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:33.862206 14329 heartbeater.cc:344] Connected to a master server at 127.13.109.190:38537
03:07:33.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:33.862826 14329 heartbeater.cc:461] Registering TS with master...
03:07:33.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:33.863494 14329 heartbeater.cc:507] Master 127.13.109.190:38537 requested a full tablet report, sending...
03:07:33.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:33.866305 13795 ts_manager.cc:194] Registered new tserver with Master: 6519e447bab048089e2f4a13a83c7d14 (127.13.109.130:43349)
03:07:33.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:33.878937 14328 heartbeater.cc:344] Connected to a master server at 127.13.109.189:33927
03:07:33.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:33.879271 14328 heartbeater.cc:461] Registering TS with master...
03:07:33.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:33.879760 14328 heartbeater.cc:507] Master 127.13.109.189:33927 requested a full tablet report, sending...
03:07:33.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:33.882787 13871 ts_manager.cc:194] Registered new tserver with Master: 6519e447bab048089e2f4a13a83c7d14 (127.13.109.130:43349)
03:07:34.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:34.155831 14337 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:34.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:34.156322 14337 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:34.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:34.156744 14337 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:34.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:34.190459 14337 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:34.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:34.191377 14337 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.13.109.131
03:07:34.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:34.227712 14337 tablet_server_runner.cc:78] Tablet server non-default flags:
03:07:34.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:34.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.109.148:36253
03:07:34.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:34.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:34.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-2/data
03:07:34.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-2/wal
03:07:34.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:34.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:34.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.109.131:0
03:07:34.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:34.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:34.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:34.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-2/data/info.pb
03:07:34.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.109.131
03:07:34.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:34.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.109.190:38537,127.13.109.189:33927,127.13.109.188:33151
03:07:34.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:34.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:34.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:34.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:34.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:34.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.109.131
03:07:34.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-2/logs
03:07:34.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:34.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:34.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:34.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:07:34.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:07:34.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
03:07:34.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:34.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Jan 2026 02:43:21 UTC on 5fd53c4cbb9d
03:07:34.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10241
03:07:34.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:34.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:34.229321 14337 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:34.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:34.231112 14337 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:34.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:34.244946 14344 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:34.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:34.854453 14327 heartbeater.cc:499] Master 127.13.109.188:33151 was elected leader, sending a full tablet report...
03:07:35.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:35.650552 14343 debug-util.cc:398] Leaking SignalData structure 0x7b080002ec60 after lost signal to thread 14337
03:07:35.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:35.966625 14337 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.722s user 0.598s sys 1.091s
03:07:35.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:34.245676 14345 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:35.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:35.967005 14337 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.722s user 0.598s sys 1.091s
03:07:35.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:35.969548 14347 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:35.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:35.971971 14346 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1721 milliseconds
03:07:35.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:35.971972 14337 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:07:35.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:35.973187 14337 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:35.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:35.975126 14337 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:35.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:35.976450 14337 hybrid_clock.cc:648] HybridClock initialized: now 1769656055976435 us; error 45 us; skew 500 ppm
03:07:35.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:35.977211 14337 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:35.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:35.983286 14337 webserver.cc:492] Webserver started at http://127.13.109.131:46335/ using document root <none> and password file <none>
03:07:35.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:35.984251 14337 fs_manager.cc:362] Metadata directory not provided
03:07:35.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:35.984445 14337 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:35.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:35.984827 14337 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
03:07:35.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:35.989048 14337 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-2/data/instance:
03:07:35.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b9e13e8428514d9b8b45a255dff14bf9"
03:07:35.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-29 03:07:35 on dist-test-slave-7fb5"
03:07:35.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:35.990350 14337 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-2/wal/instance:
03:07:35.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b9e13e8428514d9b8b45a255dff14bf9"
03:07:35.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-29 03:07:35 on dist-test-slave-7fb5"
03:07:35.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:35.997684 14337 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.001s sys 0.008s
03:07:36.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:36.004024 14355 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:36.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:36.005300 14337 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.003s
03:07:36.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:36.005612 14337 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-2/data,/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-2/wal
03:07:36.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b9e13e8428514d9b8b45a255dff14bf9"
03:07:36.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-29 03:07:35 on dist-test-slave-7fb5"
03:07:36.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:36.005908 14337 fs_report.cc:389] FS layout report
03:07:36.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:36.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-2/wal
03:07:36.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-2/wal
03:07:36.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-2/data/data
03:07:36.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:36.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:36.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:36.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:36.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:36.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:36.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:36.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:36.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:36.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:36.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:36.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:36.068171 14337 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:36.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:36.069706 14337 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:36.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:36.070109 14337 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:36.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:36.072623 14337 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
03:07:36.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:36.076785 14337 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:07:36.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:36.077004 14337 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:07:36.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:36.077196 14337 ts_tablet_manager.cc:616] Registered 0 tablets
03:07:36.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:36.077329 14337 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:07:36.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:36.220180 14337 rpc_server.cc:307] RPC server started. Bound to: 127.13.109.131:45667
03:07:36.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:36.220213 14468 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.109.131:45667 every 8 connection(s)
03:07:36.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:36.223721 14337 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-2/data/info.pb
03:07:36.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:36.234458 13750 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu as pid 14337
03:07:36.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:36.235028 13750 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster8688005024976166149/ts-2/wal/instance
03:07:36.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:36.281786 14471 heartbeater.cc:344] Connected to a master server at 127.13.109.190:38537
03:07:36.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:36.282338 14471 heartbeater.cc:461] Registering TS with master...
03:07:36.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:36.284430 14469 heartbeater.cc:344] Connected to a master server at 127.13.109.188:33151
03:07:36.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:36.284857 14469 heartbeater.cc:461] Registering TS with master...
03:07:36.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:36.286235 14469 heartbeater.cc:507] Master 127.13.109.188:33151 requested a full tablet report, sending...
03:07:36.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:36.286887 14471 heartbeater.cc:507] Master 127.13.109.190:38537 requested a full tablet report, sending...
03:07:36.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:36.288489 14470 heartbeater.cc:344] Connected to a master server at 127.13.109.189:33927
03:07:36.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:36.288820 14470 heartbeater.cc:461] Registering TS with master...
03:07:36.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:36.289433 13950 ts_manager.cc:194] Registered new tserver with Master: b9e13e8428514d9b8b45a255dff14bf9 (127.13.109.131:45667)
03:07:36.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:36.289706 14470 heartbeater.cc:507] Master 127.13.109.189:33927 requested a full tablet report, sending...
03:07:36.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:36.291244 13950 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.13.109.131:41547
03:07:36.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:36.292532 13795 ts_manager.cc:194] Registered new tserver with Master: b9e13e8428514d9b8b45a255dff14bf9 (127.13.109.131:45667)
03:07:36.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:36.294839 13871 ts_manager.cc:194] Registered new tserver with Master: b9e13e8428514d9b8b45a255dff14bf9 (127.13.109.131:45667)
03:07:36.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:36.316757 13750 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
03:07:36.331 [DEBUG - main] (MiniKuduCluster.java:195) Response:
03:07:36.334 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
03:07:36.349 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.13.109.190"
port: 38537
}
bound_http_address {
host: "127.13.109.190"
port: 41701
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.13.109.189"
port: 33927
}
bound_http_address {
host: "127.13.109.189"
port: 35627
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.13.109.188"
port: 33151
}
bound_http_address {
host: "127.13.109.188"
port: 34479
}
}
}
03:07:36.355 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
03:07:36.358 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.13.109.129"
port: 46861
}
bound_http_address {
host: "127.13.109.129"
port: 45037
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.13.109.130"
port: 43349
}
bound_http_address {
host: "127.13.109.130"
port: 36577
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.13.109.131"
port: 45667
}
bound_http_address {
host: "127.13.109.131"
port: 46335
}
}
}
03:07:36.358 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
03:07:36.545 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_275]
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51) ~[main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52) [main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41) [main/:?]
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43) [main/:?]
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155) [main/:?]
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138) [main/:?]
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431) [main/:?]
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:299) [main/:?]
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109) [main/:?]
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50) [junit-4.13.2.jar:4.13.2]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:128) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:27) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:137) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:115) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.main(JUnitCore.java:36) [junit-4.13.2.jar:4.13.2]
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.lang.Class.getDeclaredMethod(Class.java:2130)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
03:07:36.572 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
03:07:36.686 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskuwp0L7/test-tmp
03:07:36.687 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
03:07:36.687 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskuwp0L7/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
03:07:36.689 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
03:07:37.017 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:37.018 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
03:07:37.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:37.021139 14480 mini_oidc.cc:150] Starting JWKS server
03:07:37.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:37.024629 14480 webserver.cc:492] Webserver started at http://127.0.0.1:37645/ using document root <none> and password file <none>
03:07:37.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:37.025022 14480 mini_oidc.cc:186] Starting OIDC Discovery server
03:07:37.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:37.026644 14480 webserver.cc:492] Webserver started at http://127.0.0.1:44323/ using document root <none> and password file <none>
03:07:37.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
03:07:37.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/principal' for realm 'KRBTEST.COM',
03:07:37.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
03:07:37.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:07:37 dist-test-slave-7fb5 krb5kdc[14493](info): setting up network...
03:07:37.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
03:07:37.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:07:37 dist-test-slave-7fb5 krb5kdc[14493](info): set up 2 sockets
03:07:37.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:07:37 dist-test-slave-7fb5 krb5kdc[14493](info): commencing operation
03:07:37.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
03:07:37.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:37.295135 14469 heartbeater.cc:499] Master 127.13.109.188:33151 was elected leader, sending a full tablet report...
03:07:39.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:39.143217 14480 mini_kdc.cc:121] Time spent starting KDC: real 2.081s user 0.006s sys 0.013s
03:07:39.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
03:07:39.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:07:39.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
03:07:39.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
03:07:39.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:07:39.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
03:07:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
03:07:39.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:07:39.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
03:07:39.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:07:39.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/test-user.keytab.
03:07:39.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/test-user.keytab.
03:07:39.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:07:39 dist-test-slave-7fb5 krb5kdc[14493](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656059, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:39.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
03:07:39.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-01-29T03:07:39Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
03:07:39.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-01-29T03:07:39Z Disabled control of system clock
03:07:39.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.14.36.62@KRBTEST.COM; defaulting to no policy
03:07:39.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:39.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.14.36.62@KRBTEST.COM" created.
03:07:39.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:39.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.14.36.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab.
03:07:39.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.14.36.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab.
03:07:39.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.14.36.62@KRBTEST.COM; defaulting to no policy
03:07:39.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:39.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.14.36.62@KRBTEST.COM" created.
03:07:39.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:39.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.14.36.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab.
03:07:39.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.14.36.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab.
03:07:39.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:39.384617 14480 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu
03:07:39.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu
03:07:39.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/wal
03:07:39.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/data
03:07:39.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:39.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:39.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:39.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:39.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:39.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:39.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/logs
03:07:39.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/data/info.pb
03:07:39.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:39.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:39.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:39.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:39.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:39.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:39.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:39.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:39.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:07:39.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:39.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:39.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:39.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.36.62:35391
03:07:39.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.36.62
03:07:39.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:39.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab
03:07:39.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.36.62
03:07:39.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:39.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:07:39.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:07:39.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:07:39.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.36.20:43983
03:07:39.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:39.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:39.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:39.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:39.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.36.62:35391,127.14.36.61:45213,127.14.36.60:45755
03:07:39.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:39.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37645/jwks.json
03:07:39.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:07:39.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:07:39.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:39.694980 14513 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:39.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:39.695533 14513 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:39.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:39.696244 14513 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:39.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:39.729197 14513 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:07:39.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:39.729434 14513 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:07:39.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:39.729697 14513 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:39.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:39.730031 14513 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:07:39.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:39.730257 14513 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:07:39.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:39.730453 14513 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:07:39.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:39.730599 14513 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37645/jwks.json
03:07:39.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:39.768674 14513 master_runner.cc:386] Master server non-default flags:
03:07:39.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:39.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.36.20:43983
03:07:39.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:39.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:39.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/data
03:07:39.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/wal
03:07:39.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:07:39.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:39.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.36.62:35391,127.14.36.61:45213,127.14.36.60:45755
03:07:39.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab
03:07:39.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.36.62
03:07:39.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:39.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:39.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:39.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.36.62:35391
03:07:39.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:39.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:39.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:39.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37645/jwks.json
03:07:39.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:39.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:07:39.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:07:39.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:39.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:39.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/data/info.pb
03:07:39.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.36.62
03:07:39.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:39.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:39.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:39.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:39.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:39.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:39.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/logs
03:07:39.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:39.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:39.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:39.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:07:39.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:07:39.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
03:07:39.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:39.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Jan 2026 02:43:21 UTC on 5fd53c4cbb9d
03:07:39.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10241
03:07:39.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:39.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:39.770308 14513 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:39.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:39.771703 14513 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:39.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:39.782193 14519 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:41.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:41.187181 14518 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 14513
03:07:41.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:41.463989 14518 kernel_stack_watchdog.cc:198] Thread 14513 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 401ms:
03:07:41.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
03:07:41.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
03:07:41.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
03:07:41.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
03:07:41.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:41.464170 14513 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.681s user 0.532s sys 1.113s
03:07:41.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:41.464568 14513 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.681s user 0.532s sys 1.113s
03:07:41.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:39.784011 14520 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:41.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:41.466379 14522 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:41.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:41.469841 14521 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1682 milliseconds
03:07:41.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:41.469870 14513 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:07:41.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:41.471102 14513 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:41.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:41.473582 14513 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:41.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:41.474967 14513 hybrid_clock.cc:648] HybridClock initialized: now 1769656061474937 us; error 30 us; skew 500 ppm
03:07:41.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:41.476091 14513 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:41.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:07:41 dist-test-slave-7fb5 krb5kdc[14493](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656061, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:41.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:41.482589 14513 init.cc:377] Logged in from keytab as kudu/127.14.36.62@KRBTEST.COM (short username kudu)
03:07:41.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:41.487790 14513 webserver.cc:492] Webserver started at http://127.14.36.62:37233/ using document root <none> and password file <none>
03:07:41.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:41.488793 14513 fs_manager.cc:362] Metadata directory not provided
03:07:41.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:41.489048 14513 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:41.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:41.489542 14513 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
03:07:41.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:41.493888 14513 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/data/instance:
03:07:41.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6cad901ab52a4c7b9a90110bcabd529d"
03:07:41.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-29 03:07:41 on dist-test-slave-7fb5"
03:07:41.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:41.495097 14513 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/wal/instance:
03:07:41.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6cad901ab52a4c7b9a90110bcabd529d"
03:07:41.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-29 03:07:41 on dist-test-slave-7fb5"
03:07:41.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:41.502630 14513 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
03:07:41.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:41.508206 14533 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:41.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:41.509353 14513 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
03:07:41.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:41.509725 14513 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/data,/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/wal
03:07:41.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6cad901ab52a4c7b9a90110bcabd529d"
03:07:41.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-29 03:07:41 on dist-test-slave-7fb5"
03:07:41.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:41.510090 14513 fs_report.cc:389] FS layout report
03:07:41.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:41.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/wal
03:07:41.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/wal
03:07:41.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/data/data
03:07:41.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:41.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:41.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:41.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:41.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:41.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:41.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:41.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:41.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:41.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:41.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:41.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:41.562683 14513 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:41.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:41.574816 14513 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:41.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:41.575455 14513 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:41.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:41.650005 14513 rpc_server.cc:307] RPC server started. Bound to: 127.14.36.62:35391
03:07:41.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:41.650070 14585 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.14.36.62:35391 every 8 connection(s)
03:07:41.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:41.652727 14513 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/data/info.pb
03:07:41.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:41.659122 14586 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:07:41.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:41.660373 14480 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu as pid 14513
03:07:41.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:41.660847 14480 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/wal/instance
03:07:41.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:41.671062 14586 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } has no permanent_uuid. Determining permanent_uuid...
03:07:41.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.14.36.61@KRBTEST.COM; defaulting to no policy
03:07:41.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:41.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.14.36.61@KRBTEST.COM" created.
03:07:41.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:07:41 dist-test-slave-7fb5 krb5kdc[14493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656061, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.62@KRBTEST.COM for kudu/127.14.36.62@KRBTEST.COM
03:07:41.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:41.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.14.36.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab.
03:07:41.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.14.36.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab.
03:07:41.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:41.716192 14586 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } has no permanent_uuid. Determining permanent_uuid...
03:07:41.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:41.718694 14537 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.14.36.61:45213: connect: Connection refused (error 111)
03:07:41.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.14.36.61@KRBTEST.COM; defaulting to no policy
03:07:41.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:41.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.14.36.61@KRBTEST.COM" created.
03:07:41.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:41.722995 14586 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.14.36.61:45213: Network error: Client connection negotiation failed: client connection to 127.14.36.61:45213: connect: Connection refused (error 111)
03:07:41.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:41.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.14.36.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab.
03:07:41.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.14.36.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab.
03:07:41.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:41.745879 14480 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu
03:07:41.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu
03:07:41.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/wal
03:07:41.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/data
03:07:41.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:41.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:41.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:41.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:41.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:41.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:41.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/logs
03:07:41.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/data/info.pb
03:07:41.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:41.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:41.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:41.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:41.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:41.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:41.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:41.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:41.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:07:41.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:41.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:41.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:41.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.36.61:45213
03:07:41.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.36.61
03:07:41.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:41.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab
03:07:41.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.36.61
03:07:41.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:41.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:07:41.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:07:41.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:07:41.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.36.20:43983
03:07:41.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:41.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:41.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:41.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:41.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.36.62:35391,127.14.36.61:45213,127.14.36.60:45755
03:07:41.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:41.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37645/jwks.json
03:07:41.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:07:41.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:07:41.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:41.772535 14586 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } attempt: 1
03:07:41.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:41.776715 14586 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.14.36.61:45213: Network error: Client connection negotiation failed: client connection to 127.14.36.61:45213: connect: Connection refused (error 111)
03:07:41.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:41.845104 14586 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } attempt: 2
03:07:41.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:41.849074 14586 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.14.36.61:45213: Network error: Client connection negotiation failed: client connection to 127.14.36.61:45213: connect: Connection refused (error 111)
03:07:41.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:41.940551 14586 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } attempt: 3
03:07:41.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:41.944519 14586 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.14.36.61:45213: Network error: Client connection negotiation failed: client connection to 127.14.36.61:45213: connect: Connection refused (error 111)
03:07:42.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:42.057163 14596 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:42.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:42.057693 14596 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:42.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:42.058609 14596 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:42.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:42.088011 14586 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } attempt: 4
03:07:42.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:42.091746 14586 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.14.36.61:45213: Network error: Client connection negotiation failed: client connection to 127.14.36.61:45213: connect: Connection refused (error 111)
03:07:42.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:42.092993 14596 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:07:42.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:42.093220 14596 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:07:42.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:42.093493 14596 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:42.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:42.093688 14596 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:07:42.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:42.093883 14596 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:07:42.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:42.094075 14596 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:07:42.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:42.094229 14596 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37645/jwks.json
03:07:42.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:42.138207 14596 master_runner.cc:386] Master server non-default flags:
03:07:42.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:42.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.36.20:43983
03:07:42.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:42.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:42.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/data
03:07:42.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/wal
03:07:42.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:07:42.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:42.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.36.62:35391,127.14.36.61:45213,127.14.36.60:45755
03:07:42.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab
03:07:42.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.36.61
03:07:42.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:42.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:42.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:42.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.36.61:45213
03:07:42.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:42.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:42.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:42.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37645/jwks.json
03:07:42.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:42.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:07:42.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:07:42.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:42.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:42.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/data/info.pb
03:07:42.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.36.61
03:07:42.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:42.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:42.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:42.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:42.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:42.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:42.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/logs
03:07:42.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:42.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:42.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:42.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:07:42.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:07:42.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
03:07:42.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:42.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Jan 2026 02:43:21 UTC on 5fd53c4cbb9d
03:07:42.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10241
03:07:42.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:42.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:42.139869 14596 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:42.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:42.141258 14596 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:42.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:42.153071 14603 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:42.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:42.391249 14586 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } attempt: 5
03:07:42.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:42.398495 14586 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.14.36.61:45213: Network error: Client connection negotiation failed: client connection to 127.14.36.61:45213: connect: Connection refused (error 111)
03:07:42.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:42.946075 14586 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } attempt: 6
03:07:42.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:42.968324 14586 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.14.36.61:45213: Network error: Client connection negotiation failed: client connection to 127.14.36.61:45213: connect: Connection refused (error 111)
03:07:43.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:42.154340 14604 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:43.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:43.517052 14606 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:43.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:43.519464 14605 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1361 milliseconds
03:07:43.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:43.519634 14596 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:07:43.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:43.522722 14596 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:43.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:43.526516 14596 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:43.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:43.527989 14596 hybrid_clock.cc:648] HybridClock initialized: now 1769656063527910 us; error 79 us; skew 500 ppm
03:07:43.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:43.529505 14596 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:43.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:07:43 dist-test-slave-7fb5 krb5kdc[14493](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656063, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:43.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:43.536157 14596 init.cc:377] Logged in from keytab as kudu/127.14.36.61@KRBTEST.COM (short username kudu)
03:07:43.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:43.542766 14596 webserver.cc:492] Webserver started at http://127.14.36.61:44571/ using document root <none> and password file <none>
03:07:43.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:43.543807 14596 fs_manager.cc:362] Metadata directory not provided
03:07:43.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:43.544148 14596 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:43.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:43.544692 14596 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
03:07:43.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:43.549279 14596 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/data/instance:
03:07:43.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "de1dd59a869a46bdb0165c8963557689"
03:07:43.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-29 03:07:43 on dist-test-slave-7fb5"
03:07:43.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:43.550549 14596 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/wal/instance:
03:07:43.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "de1dd59a869a46bdb0165c8963557689"
03:07:43.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-29 03:07:43 on dist-test-slave-7fb5"
03:07:43.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:43.559289 14596 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.005s sys 0.004s
03:07:43.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:43.565416 14617 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:43.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:43.566579 14596 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
03:07:43.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:43.567035 14596 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/data,/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/wal
03:07:43.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "de1dd59a869a46bdb0165c8963557689"
03:07:43.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-29 03:07:43 on dist-test-slave-7fb5"
03:07:43.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:43.567409 14596 fs_report.cc:389] FS layout report
03:07:43.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:43.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/wal
03:07:43.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/wal
03:07:43.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/data/data
03:07:43.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:43.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:43.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:43.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:43.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:43.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:43.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:43.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:43.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:43.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:43.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:43.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:43.623526 14596 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:43.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:43.629606 14596 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:43.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:43.630141 14596 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:43.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:43.703612 14596 rpc_server.cc:307] RPC server started. Bound to: 127.14.36.61:45213
03:07:43.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:43.703684 14668 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.14.36.61:45213 every 8 connection(s)
03:07:43.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:43.706440 14596 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/data/info.pb
03:07:43.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:43.713024 14669 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:07:43.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:43.715910 14480 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu as pid 14596
03:07:43.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:43.716391 14480 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/wal/instance
03:07:43.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:43.727655 14669 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } has no permanent_uuid. Determining permanent_uuid...
03:07:43.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.14.36.60@KRBTEST.COM; defaulting to no policy
03:07:43.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:43.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.14.36.60@KRBTEST.COM" created.
03:07:43.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:07:43 dist-test-slave-7fb5 krb5kdc[14493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656063, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.61@KRBTEST.COM for kudu/127.14.36.62@KRBTEST.COM
03:07:43.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:43.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.14.36.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab.
03:07:43.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.14.36.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab.
03:07:43.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:43.764678 14669 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } has no permanent_uuid. Determining permanent_uuid...
03:07:43.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:07:43 dist-test-slave-7fb5 krb5kdc[14493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656063, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.61@KRBTEST.COM for kudu/127.14.36.61@KRBTEST.COM
03:07:43.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.14.36.60@KRBTEST.COM; defaulting to no policy
03:07:43.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:43.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.14.36.60@KRBTEST.COM" created.
03:07:43.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:43.790808 14669 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } has no permanent_uuid. Determining permanent_uuid...
03:07:43.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:43.792972 14621 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.14.36.60:45755: connect: Connection refused (error 111)
03:07:43.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:43.797562 14669 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.14.36.60:45755: Network error: Client connection negotiation failed: client connection to 127.14.36.60:45755: connect: Connection refused (error 111)
03:07:43.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:43.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.14.36.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab.
03:07:43.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.14.36.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab.
03:07:43.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:43.807732 14480 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu
03:07:43.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu
03:07:43.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/wal
03:07:43.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/data
03:07:43.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:43.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:43.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:43.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:43.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:43.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:43.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/logs
03:07:43.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/data/info.pb
03:07:43.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:43.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:43.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:43.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:43.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:43.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:43.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:43.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:43.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:07:43.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:43.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:43.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:43.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.36.60:45755
03:07:43.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.36.60
03:07:43.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:43.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab
03:07:43.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.36.60
03:07:43.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:43.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:07:43.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:07:43.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:07:43.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.36.20:43983
03:07:43.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:43.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:43.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:43.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:43.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.36.62:35391,127.14.36.61:45213,127.14.36.60:45755
03:07:43.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:43.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37645/jwks.json
03:07:43.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:07:43.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:07:43.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:43.847064 14669 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } attempt: 1
03:07:43.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:43.851600 14669 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.14.36.60:45755: Network error: Client connection negotiation failed: client connection to 127.14.36.60:45755: connect: Connection refused (error 111)
03:07:43.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:43.920195 14669 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } attempt: 2
03:07:43.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:43.924757 14669 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.14.36.60:45755: Network error: Client connection negotiation failed: client connection to 127.14.36.60:45755: connect: Connection refused (error 111)
03:07:44.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:44.016219 14669 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } attempt: 3
03:07:44.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:44.020517 14669 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.14.36.60:45755: Network error: Client connection negotiation failed: client connection to 127.14.36.60:45755: connect: Connection refused (error 111)
03:07:44.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:44.028903 14586 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } attempt: 7
03:07:44.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:07:44 dist-test-slave-7fb5 krb5kdc[14493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656061, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.62@KRBTEST.COM for kudu/127.14.36.61@KRBTEST.COM
03:07:44.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:44.052414 14586 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } has no permanent_uuid. Determining permanent_uuid...
03:07:44.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:44.057569 14586 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.14.36.60:45755: Network error: Client connection negotiation failed: client connection to 127.14.36.60:45755: connect: Connection refused (error 111)
03:07:44.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:44.115968 14586 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } attempt: 1
03:07:44.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:44.119576 14586 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.14.36.60:45755: Network error: Client connection negotiation failed: client connection to 127.14.36.60:45755: connect: Connection refused (error 111)
03:07:44.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:44.129230 14679 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:44.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:44.129834 14679 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:44.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:44.130493 14679 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:44.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:44.164014 14669 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } attempt: 4
03:07:44.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:44.167940 14669 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.14.36.60:45755: Network error: Client connection negotiation failed: client connection to 127.14.36.60:45755: connect: Connection refused (error 111)
03:07:44.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:44.174010 14679 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:07:44.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:44.174253 14679 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:07:44.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:44.174516 14679 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:44.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:44.174711 14679 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:07:44.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:44.174893 14679 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:07:44.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:44.175086 14679 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:07:44.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:44.175241 14679 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37645/jwks.json
03:07:44.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:44.201028 14586 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } attempt: 2
03:07:44.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:44.205400 14586 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.14.36.60:45755: Network error: Client connection negotiation failed: client connection to 127.14.36.60:45755: connect: Connection refused (error 111)
03:07:44.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:44.215396 14679 master_runner.cc:386] Master server non-default flags:
03:07:44.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:44.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.36.20:43983
03:07:44.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:44.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:44.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/data
03:07:44.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/wal
03:07:44.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:07:44.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:44.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.36.62:35391,127.14.36.61:45213,127.14.36.60:45755
03:07:44.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab
03:07:44.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.36.60
03:07:44.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:44.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:44.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:44.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.36.60:45755
03:07:44.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:44.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:44.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:44.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37645/jwks.json
03:07:44.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:44.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:07:44.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:07:44.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:44.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:44.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/data/info.pb
03:07:44.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.36.60
03:07:44.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:44.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:44.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:44.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:44.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:44.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:44.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/logs
03:07:44.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:44.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:44.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:44.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:07:44.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:07:44.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
03:07:44.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:44.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Jan 2026 02:43:21 UTC on 5fd53c4cbb9d
03:07:44.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10241
03:07:44.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:44.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:44.216722 14679 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:44.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:44.218029 14679 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:44.232599 14690 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:44.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:44.290841 14586 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } attempt: 3
03:07:44.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:44.298382 14586 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.14.36.60:45755: Network error: Client connection negotiation failed: client connection to 127.14.36.60:45755: connect: Connection refused (error 111)
03:07:44.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:44.438830 14586 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } attempt: 4
03:07:44.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:44.446763 14586 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.14.36.60:45755: Network error: Client connection negotiation failed: client connection to 127.14.36.60:45755: connect: Connection refused (error 111)
03:07:44.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:44.467432 14669 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } attempt: 5
03:07:44.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:44.476042 14669 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.14.36.60:45755: Network error: Client connection negotiation failed: client connection to 127.14.36.60:45755: connect: Connection refused (error 111)
03:07:44.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:44.730293 14586 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } attempt: 5
03:07:44.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:44.737505 14586 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.14.36.60:45755: Network error: Client connection negotiation failed: client connection to 127.14.36.60:45755: connect: Connection refused (error 111)
03:07:45.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:45.023638 14669 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } attempt: 6
03:07:45.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:45.060568 14669 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.14.36.60:45755: Network error: Client connection negotiation failed: client connection to 127.14.36.60:45755: connect: Connection refused (error 111)
03:07:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:45.290108 14586 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } attempt: 6
03:07:45.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:45.312999 14586 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.14.36.60:45755: Network error: Client connection negotiation failed: client connection to 127.14.36.60:45755: connect: Connection refused (error 111)
03:07:45.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:44.236853 14691 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:45.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:45.636991 14689 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 14679
03:07:45.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:45.710368 14679 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.474s user 0.453s sys 0.945s
03:07:45.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:45.712013 14692 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1474 milliseconds
03:07:45.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:45.713208 14693 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:45.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:45.713306 14679 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.477s user 0.453s sys 0.945s
03:07:45.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:45.713610 14679 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:07:45.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:45.719310 14679 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:45.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:45.722057 14679 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:45.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:45.723471 14679 hybrid_clock.cc:648] HybridClock initialized: now 1769656065723435 us; error 41 us; skew 500 ppm
03:07:45.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:45.724452 14679 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:45.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:07:45 dist-test-slave-7fb5 krb5kdc[14493](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656065, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:45.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:45.730455 14679 init.cc:377] Logged in from keytab as kudu/127.14.36.60@KRBTEST.COM (short username kudu)
03:07:45.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:45.735688 14679 webserver.cc:492] Webserver started at http://127.14.36.60:41699/ using document root <none> and password file <none>
03:07:45.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:45.736665 14679 fs_manager.cc:362] Metadata directory not provided
03:07:45.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:45.736907 14679 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:45.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:45.737433 14679 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
03:07:45.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:45.742069 14679 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/data/instance:
03:07:45.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a5feff59896e4438ac3b32435885f30c"
03:07:45.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-29 03:07:45 on dist-test-slave-7fb5"
03:07:45.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:45.743249 14679 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/wal/instance:
03:07:45.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a5feff59896e4438ac3b32435885f30c"
03:07:45.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-29 03:07:45 on dist-test-slave-7fb5"
03:07:45.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:45.750838 14679 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.001s
03:07:45.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:45.757443 14706 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:45.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:45.758579 14679 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.000s sys 0.005s
03:07:45.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:45.758998 14679 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/data,/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/wal
03:07:45.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a5feff59896e4438ac3b32435885f30c"
03:07:45.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-29 03:07:45 on dist-test-slave-7fb5"
03:07:45.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:45.759378 14679 fs_report.cc:389] FS layout report
03:07:45.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:45.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/wal
03:07:45.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/wal
03:07:45.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/data/data
03:07:45.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:45.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:45.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:45.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:45.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:45.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:45.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:45.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:45.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:45.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:45.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:45.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:45.808789 14679 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:45.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:45.814680 14679 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:45.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:45.815207 14679 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:45.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:45.888104 14679 rpc_server.cc:307] RPC server started. Bound to: 127.14.36.60:45755
03:07:45.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:45.888123 14757 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.14.36.60:45755 every 8 connection(s)
03:07:45.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:45.890944 14679 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/data/info.pb
03:07:45.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:45.896944 14758 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:07:45.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:45.899315 14480 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu as pid 14679
03:07:45.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:45.899812 14480 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/wal/instance
03:07:45.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:45.914407 14758 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } has no permanent_uuid. Determining permanent_uuid...
03:07:45.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:07:45 dist-test-slave-7fb5 krb5kdc[14493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656059, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.14.36.62@KRBTEST.COM
03:07:45.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:07:45 dist-test-slave-7fb5 krb5kdc[14493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656065, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.60@KRBTEST.COM for kudu/127.14.36.62@KRBTEST.COM
03:07:45.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:45.953719 14758 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } has no permanent_uuid. Determining permanent_uuid...
03:07:45.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:07:45 dist-test-slave-7fb5 krb5kdc[14493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656059, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.14.36.61@KRBTEST.COM
03:07:45.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:07:45 dist-test-slave-7fb5 krb5kdc[14493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656065, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.60@KRBTEST.COM for kudu/127.14.36.61@KRBTEST.COM
03:07:45.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:45.977815 14758 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } has no permanent_uuid. Determining permanent_uuid...
03:07:45.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:07:45 dist-test-slave-7fb5 krb5kdc[14493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656059, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.14.36.60@KRBTEST.COM
03:07:45.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:07:45 dist-test-slave-7fb5 krb5kdc[14493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656065, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.60@KRBTEST.COM for kudu/127.14.36.60@KRBTEST.COM
03:07:46.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.007643 14758 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c: Bootstrap starting.
03:07:46.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.013435 14758 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c: Neither blocks nor log segments found. Creating new log.
03:07:46.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.015940 14758 log.cc:826] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c: Log is configured to *not* fsync() on all Append() calls
03:07:46.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.020550 14758 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c: No bootstrap required, opened a new log
03:07:46.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.040141 14758 raft_consensus.cc:359] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } }
03:07:46.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.040850 14758 raft_consensus.cc:385] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:07:46.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.041062 14758 raft_consensus.cc:740] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a5feff59896e4438ac3b32435885f30c, State: Initialized, Role: FOLLOWER
03:07:46.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.041800 14758 consensus_queue.cc:260] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } }
03:07:46.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.044567 14769 sys_catalog.cc:455] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } } }
03:07:46.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.045188 14769 sys_catalog.cc:458] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c [sys.catalog]: This master's current role is: FOLLOWER
03:07:46.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.046265 14758 sys_catalog.cc:565] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c [sys.catalog]: configured and running, proceeding with master startup.
03:07:46.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:46.070330 14712 tablet.cc:2378] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:07:46.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:46.072507 14781 catalog_manager.cc:1568] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:07:46.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:46.072847 14781 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:07:46.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.068207 14679 master_runner.cc:186] Error getting master registration for 127.14.36.62:35391: OK, instance_id { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" instance_seqno: 1769656061629713 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:07:46.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.092482 14679 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:07:46.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.121187 14669 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } attempt: 7
03:07:46.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:07:46 dist-test-slave-7fb5 krb5kdc[14493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656063, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.61@KRBTEST.COM for kudu/127.14.36.60@KRBTEST.COM
03:07:46.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.153973 14669 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689: Bootstrap starting.
03:07:46.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.161433 14669 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689: Neither blocks nor log segments found. Creating new log.
03:07:46.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.165112 14669 log.cc:826] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689: Log is configured to *not* fsync() on all Append() calls
03:07:46.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.174430 14669 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689: No bootstrap required, opened a new log
03:07:46.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.194442 14669 raft_consensus.cc:359] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } }
03:07:46.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.195086 14669 raft_consensus.cc:385] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:07:46.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.195278 14669 raft_consensus.cc:740] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: de1dd59a869a46bdb0165c8963557689, State: Initialized, Role: FOLLOWER
03:07:46.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.196254 14669 consensus_queue.cc:260] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } }
03:07:46.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.199819 14785 sys_catalog.cc:455] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } } }
03:07:46.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.201007 14785 sys_catalog.cc:458] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [sys.catalog]: This master's current role is: FOLLOWER
03:07:46.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.201836 14669 sys_catalog.cc:565] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [sys.catalog]: configured and running, proceeding with master startup.
03:07:46.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.208408 14785 raft_consensus.cc:493] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:07:46.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.208926 14785 raft_consensus.cc:515] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } }
03:07:46.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.211840 14785 leader_election.cc:290] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6cad901ab52a4c7b9a90110bcabd529d (127.14.36.62:35391), a5feff59896e4438ac3b32435885f30c (127.14.36.60:45755)
03:07:46.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.213464 14733 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "de1dd59a869a46bdb0165c8963557689" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a5feff59896e4438ac3b32435885f30c" is_pre_election: true
03:07:46.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.214510 14733 raft_consensus.cc:2468] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate de1dd59a869a46bdb0165c8963557689 in term 0.
03:07:46.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.220391 14621 leader_election.cc:304] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a5feff59896e4438ac3b32435885f30c, de1dd59a869a46bdb0165c8963557689; no voters:
03:07:46.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.222029 14785 raft_consensus.cc:2804] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [term 0 FOLLOWER]: Leader pre-election won for term 1
03:07:46.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.222507 14785 raft_consensus.cc:493] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:07:46.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.222942 14785 raft_consensus.cc:3060] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [term 0 FOLLOWER]: Advancing to term 1
03:07:46.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.237457 14785 raft_consensus.cc:515] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } }
03:07:46.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.241662 14733 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "de1dd59a869a46bdb0165c8963557689" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a5feff59896e4438ac3b32435885f30c"
03:07:46.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.242411 14733 raft_consensus.cc:3060] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c [term 0 FOLLOWER]: Advancing to term 1
03:07:46.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.259007 14785 leader_election.cc:290] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [CANDIDATE]: Term 1 election: Requested vote from peers 6cad901ab52a4c7b9a90110bcabd529d (127.14.36.62:35391), a5feff59896e4438ac3b32435885f30c (127.14.36.60:45755)
03:07:46.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.260995 14733 raft_consensus.cc:2468] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate de1dd59a869a46bdb0165c8963557689 in term 1.
03:07:46.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.270702 14621 leader_election.cc:304] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a5feff59896e4438ac3b32435885f30c, de1dd59a869a46bdb0165c8963557689; no voters:
03:07:46.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.274456 14785 raft_consensus.cc:2804] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [term 1 FOLLOWER]: Leader election won for term 1
03:07:46.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.277978 14785 raft_consensus.cc:697] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [term 1 LEADER]: Becoming Leader. State: Replica: de1dd59a869a46bdb0165c8963557689, State: Running, Role: LEADER
03:07:46.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.279767 14785 consensus_queue.cc:237] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } }
03:07:46.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.305068 14792 sys_catalog.cc:455] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [sys.catalog]: SysCatalogTable state changed. Reason: New leader de1dd59a869a46bdb0165c8963557689. Latest consensus state: current_term: 1 leader_uuid: "de1dd59a869a46bdb0165c8963557689" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } } }
03:07:46.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.306206 14792 sys_catalog.cc:458] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [sys.catalog]: This master's current role is: LEADER
03:07:46.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:46.341176 14798 catalog_manager.cc:1568] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:07:46.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:46.342197 14798 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:07:46.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.345495 14799 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
03:07:46.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.346486 14586 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } attempt: 7
03:07:46.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.388896 14799 catalog_manager.cc:1494] Initializing Kudu cluster ID...
03:07:46.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:07:46 dist-test-slave-7fb5 krb5kdc[14493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656061, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.62@KRBTEST.COM for kudu/127.14.36.60@KRBTEST.COM
03:07:46.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:46.390218 14623 tablet.cc:2378] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:07:46.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.313704 14596 master_runner.cc:186] Error getting master registration for 127.14.36.62:35391: OK, instance_id { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" instance_seqno: 1769656061629713 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:07:46.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.427827 14733 raft_consensus.cc:1275] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c [term 1 FOLLOWER]: Refusing update from remote peer de1dd59a869a46bdb0165c8963557689: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:07:46.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.428968 14596 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:07:46.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.431036 14792 consensus_queue.cc:1048] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:07:46.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.488212 14586 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d: Bootstrap starting.
03:07:46.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.497489 14769 sys_catalog.cc:455] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c [sys.catalog]: SysCatalogTable state changed. Reason: New leader de1dd59a869a46bdb0165c8963557689. Latest consensus state: current_term: 1 leader_uuid: "de1dd59a869a46bdb0165c8963557689" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } } }
03:07:46.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.497994 14802 mvcc.cc:204] Tried to move back new op lower bound from 7248511248071487488 to 7248511247501586432. Current Snapshot: MvccSnapshot[applied={T|T < 7248511248071487488}]
03:07:46.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.498225 14769 sys_catalog.cc:458] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c [sys.catalog]: This master's current role is: FOLLOWER
03:07:46.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.501051 14785 sys_catalog.cc:455] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "de1dd59a869a46bdb0165c8963557689" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } } }
03:07:46.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.501776 14785 sys_catalog.cc:458] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [sys.catalog]: This master's current role is: LEADER
03:07:46.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.502851 14803 mvcc.cc:204] Tried to move back new op lower bound from 7248511248071487488 to 7248511247501586432. Current Snapshot: MvccSnapshot[applied={T|T < 7248511248071487488}]
03:07:46.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.506399 14586 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d: Neither blocks nor log segments found. Creating new log.
03:07:46.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.508944 14586 log.cc:826] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d: Log is configured to *not* fsync() on all Append() calls
03:07:46.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.513715 14769 sys_catalog.cc:455] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "de1dd59a869a46bdb0165c8963557689" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } } }
03:07:46.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.514454 14769 sys_catalog.cc:458] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c [sys.catalog]: This master's current role is: FOLLOWER
03:07:46.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.517652 14799 catalog_manager.cc:1357] Generated new cluster ID: 5d811d19a7434727a35b22e3a8e4d9bd
03:07:46.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.517959 14799 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
03:07:46.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.552888 14586 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d: No bootstrap required, opened a new log
03:07:46.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.581437 14799 catalog_manager.cc:1380] Generated new certificate authority record
03:07:46.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.583499 14799 catalog_manager.cc:1514] Loading token signing keys...
03:07:46.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.609392 14586 raft_consensus.cc:359] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } }
03:07:46.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.610621 14586 raft_consensus.cc:385] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:07:46.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.611080 14586 raft_consensus.cc:740] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6cad901ab52a4c7b9a90110bcabd529d, State: Initialized, Role: FOLLOWER
03:07:46.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.612249 14586 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } }
03:07:46.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.626541 14561 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "de1dd59a869a46bdb0165c8963557689" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6cad901ab52a4c7b9a90110bcabd529d" is_pre_election: true
03:07:46.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.641734 14560 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "de1dd59a869a46bdb0165c8963557689" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6cad901ab52a4c7b9a90110bcabd529d"
03:07:46.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.643251 14808 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } } }
03:07:46.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.644104 14808 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [sys.catalog]: This master's current role is: FOLLOWER
03:07:46.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.644721 14559 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [term 0 FOLLOWER]: Advancing to term 1
03:07:46.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.656154 14799 catalog_manager.cc:6027] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689: Generated new TSK 0
03:07:46.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.654075 14586 sys_catalog.cc:565] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [sys.catalog]: configured and running, proceeding with master startup.
03:07:46.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.657437 14799 catalog_manager.cc:1524] Initializing in-progress tserver states...
03:07:46.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.660079 14559 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [term 1 FOLLOWER]: Refusing update from remote peer de1dd59a869a46bdb0165c8963557689: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:07:46.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.663988 14785 consensus_queue.cc:1048] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:07:46.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.14.36.1@KRBTEST.COM; defaulting to no policy
03:07:46.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:46.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.14.36.1@KRBTEST.COM" created.
03:07:46.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.794065 14821 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "de1dd59a869a46bdb0165c8963557689" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } } }
03:07:46.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.796533 14821 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [sys.catalog]: This master's current role is: FOLLOWER
03:07:46.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.795843 14808 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [sys.catalog]: SysCatalogTable state changed. Reason: New leader de1dd59a869a46bdb0165c8963557689. Latest consensus state: current_term: 1 leader_uuid: "de1dd59a869a46bdb0165c8963557689" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } } }
03:07:46.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.797380 14808 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [sys.catalog]: This master's current role is: FOLLOWER
03:07:46.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:46.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.14.36.1 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab.
03:07:46.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.14.36.1 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab.
03:07:46.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:46.811018 14480 mini_kdc.cc:268] Time spent creating service keytab for kudu/127.14.36.1: real 0.112s user 0.002s sys 0.012s
03:07:46.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.850376 14826 catalog_manager.cc:1269] Loaded cluster ID: 5d811d19a7434727a35b22e3a8e4d9bd
03:07:46.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.850674 14826 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d: loading cluster ID for follower catalog manager: success
03:07:46.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.858160 14826 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d: acquiring CA information for follower catalog manager: success
03:07:46.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.14.36.1@KRBTEST.COM; defaulting to no policy
03:07:46.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:46.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.14.36.1@KRBTEST.COM" created.
03:07:46.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.876335 14826 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:07:46.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:46.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.14.36.1 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab.
03:07:46.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.14.36.1 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab.
03:07:46.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:46.915832 14480 mini_kdc.cc:268] Time spent creating service keytab for HTTP/127.14.36.1: real 0.104s user 0.005s sys 0.007s
03:07:46.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:46.918195 14480 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu
03:07:46.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu
03:07:46.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-0/wal
03:07:46.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-0/data
03:07:46.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:46.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:46.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:46.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:46.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:46.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:46.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-0/logs
03:07:46.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-0/data/info.pb
03:07:46.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:46.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:46.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:46.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:46.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:46.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:46.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:46.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:46.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:07:46.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:46.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.36.1:0
03:07:46.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.14.36.1
03:07:46.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.36.1
03:07:46.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:46.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.14.36.62:35391,127.14.36.61:45213,127.14.36.60:45755
03:07:46.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab
03:07:46.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.36.1
03:07:46.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:46.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:07:46.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:07:46.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:07:46.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.36.20:43983
03:07:46.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:46.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:46.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:46.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:07:47.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:47.078681 14781 catalog_manager.cc:1269] Loaded cluster ID: 5d811d19a7434727a35b22e3a8e4d9bd
03:07:47.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:47.079245 14781 catalog_manager.cc:1562] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c: loading cluster ID for follower catalog manager: success
03:07:47.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:47.091377 14781 catalog_manager.cc:1584] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c: acquiring CA information for follower catalog manager: success
03:07:47.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:47.104449 14781 catalog_manager.cc:1612] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:07:47.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:47.264178 14785 sys_catalog.cc:455] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "de1dd59a869a46bdb0165c8963557689" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } } }
03:07:47.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:47.264951 14785 sys_catalog.cc:458] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [sys.catalog]: This master's current role is: LEADER
03:07:47.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:47.672623 14830 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:47.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:47.673589 14830 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:47.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:47.674758 14830 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:47.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:47.734826 14830 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:47.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:47.736824 14830 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.14.36.1
03:07:47.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:47.809324 14830 tablet_server_runner.cc:78] Tablet server non-default flags:
03:07:47.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:47.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.36.20:43983
03:07:47.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:47.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:47.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-0/data
03:07:47.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-0/wal
03:07:47.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab
03:07:47.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.36.1
03:07:47.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:47.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:47.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.36.1:0
03:07:47.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:47.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:47.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:07:47.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:07:47.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:47.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:47.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-0/data/info.pb
03:07:47.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.36.1
03:07:47.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:47.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.14.36.62:35391,127.14.36.61:45213,127.14.36.60:45755
03:07:47.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:47.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:47.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:47.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:47.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:47.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.14.36.1
03:07:47.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-0/logs
03:07:47.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:47.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:47.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:47.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:07:47.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:07:47.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
03:07:47.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:47.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Jan 2026 02:43:21 UTC on 5fd53c4cbb9d
03:07:47.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10241
03:07:47.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:47.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:47.814043 14830 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:47.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:47.815761 14830 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:47.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:47.855062 14839 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:49.247612 14838 debug-util.cc:398] Leaking SignalData structure 0x7b08000282a0 after lost signal to thread 14830
03:07:50.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:50.174144 14838 kernel_stack_watchdog.cc:198] Thread 14830 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 381ms:
03:07:50.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
03:07:50.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
03:07:50.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
03:07:50.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
03:07:50.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:47.866655 14840 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:50.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:50.178231 14830 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.314s user 0.563s sys 0.736s
03:07:50.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:50.178609 14830 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.314s user 0.563s sys 0.736s
03:07:50.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:50.188230 14830 server_base.cc:1060] running on GCE node
03:07:50.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:50.189095 14846 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:50.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:50.190476 14830 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:50.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:50.193333 14830 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:50.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:50.196913 14830 hybrid_clock.cc:648] HybridClock initialized: now 1769656070196601 us; error 309 us; skew 500 ppm
03:07:50.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:50.197960 14830 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:50.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:07:50 dist-test-slave-7fb5 krb5kdc[14493](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656070, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.1@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:50.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:50.205943 14830 init.cc:377] Logged in from keytab as kudu/127.14.36.1@KRBTEST.COM (short username kudu)
03:07:50.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:50.284363 14830 webserver.cc:492] Webserver started at http://127.14.36.1:41393/ using document root <none> and password file <none>
03:07:50.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:50.286036 14830 fs_manager.cc:362] Metadata directory not provided
03:07:50.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:50.286541 14830 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:50.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:50.287181 14830 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
03:07:50.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:50.293848 14830 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-0/data/instance:
03:07:50.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1e74ae69837f4a19929b3bf70017a23d"
03:07:50.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-29 03:07:50 on dist-test-slave-7fb5"
03:07:50.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:50.295778 14830 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-0/wal/instance:
03:07:50.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1e74ae69837f4a19929b3bf70017a23d"
03:07:50.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-29 03:07:50 on dist-test-slave-7fb5"
03:07:50.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:50.307128 14830 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.006s sys 0.003s
03:07:50.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:50.315351 14852 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:50.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:50.316905 14830 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.006s sys 0.000s
03:07:50.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:50.318818 14830 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-0/data,/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-0/wal
03:07:50.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1e74ae69837f4a19929b3bf70017a23d"
03:07:50.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-29 03:07:50 on dist-test-slave-7fb5"
03:07:50.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:50.319576 14830 fs_report.cc:389] FS layout report
03:07:50.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:50.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-0/wal
03:07:50.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-0/wal
03:07:50.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-0/data/data
03:07:50.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:50.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:50.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:50.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:50.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:50.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:50.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:50.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:50.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:50.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:50.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:50.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:50.385166 14830 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:50.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:50.392422 14830 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:50.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:50.392966 14830 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:50.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:50.395742 14830 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
03:07:50.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:50.403278 14830 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:07:50.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:50.403574 14830 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:07:50.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:50.404484 14830 ts_tablet_manager.cc:616] Registered 0 tablets
03:07:50.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:50.404735 14830 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s user 0.001s sys 0.000s
03:07:50.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:50.566407 14966 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.14.36.1:45385 every 8 connection(s)
03:07:50.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:50.566929 14830 rpc_server.cc:307] RPC server started. Bound to: 127.14.36.1:45385
03:07:50.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:50.570428 14830 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-0/data/info.pb
03:07:50.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:50.572310 14480 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu as pid 14830
03:07:50.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:50.572917 14480 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-0/wal/instance
03:07:50.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.14.36.2@KRBTEST.COM; defaulting to no policy
03:07:50.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:50.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.14.36.2@KRBTEST.COM" created.
03:07:50.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:07:50 dist-test-slave-7fb5 krb5kdc[14493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656070, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.1@KRBTEST.COM for kudu/127.14.36.62@KRBTEST.COM
03:07:50.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:07:50 dist-test-slave-7fb5 krb5kdc[14493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656070, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.1@KRBTEST.COM for kudu/127.14.36.60@KRBTEST.COM
03:07:50.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:07:50 dist-test-slave-7fb5 krb5kdc[14493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656070, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.1@KRBTEST.COM for kudu/127.14.36.61@KRBTEST.COM
03:07:50.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:50.643402 14969 heartbeater.cc:344] Connected to a master server at 127.14.36.62:35391
03:07:50.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:50.644049 14969 heartbeater.cc:461] Registering TS with master...
03:07:50.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:50.645510 14969 heartbeater.cc:507] Master 127.14.36.62:35391 requested a full tablet report, sending...
03:07:50.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:50.646509 14967 heartbeater.cc:344] Connected to a master server at 127.14.36.60:45755
03:07:50.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:50.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.14.36.2 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab.
03:07:50.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.14.36.2 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab.
03:07:50.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:50.646934 14967 heartbeater.cc:461] Registering TS with master...
03:07:50.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:50.648910 14968 heartbeater.cc:344] Connected to a master server at 127.14.36.61:45213
03:07:50.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:50.649247 14968 heartbeater.cc:461] Registering TS with master...
03:07:50.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:50.650198 14968 heartbeater.cc:507] Master 127.14.36.61:45213 requested a full tablet report, sending...
03:07:50.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:50.654631 14967 heartbeater.cc:507] Master 127.14.36.60:45755 requested a full tablet report, sending...
03:07:50.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:50.655262 14551 ts_manager.cc:194] Registered new tserver with Master: 1e74ae69837f4a19929b3bf70017a23d (127.14.36.1:45385)
03:07:50.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:50.658680 14723 ts_manager.cc:194] Registered new tserver with Master: 1e74ae69837f4a19929b3bf70017a23d (127.14.36.1:45385)
03:07:50.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:50.658684 14634 ts_manager.cc:194] Registered new tserver with Master: 1e74ae69837f4a19929b3bf70017a23d (127.14.36.1:45385)
03:07:50.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:50.661456 14634 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.14.36.1@KRBTEST.COM'} at 127.14.36.1:54941
03:07:50.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.14.36.2@KRBTEST.COM; defaulting to no policy
03:07:50.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:50.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.14.36.2@KRBTEST.COM" created.
03:07:50.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:50.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.14.36.2 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab.
03:07:50.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.14.36.2 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab.
03:07:50.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:50.694710 14480 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu
03:07:50.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu
03:07:50.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-1/wal
03:07:50.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-1/data
03:07:50.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:50.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:50.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:50.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:50.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:50.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:50.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-1/logs
03:07:50.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-1/data/info.pb
03:07:50.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:50.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:50.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:50.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:50.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:50.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:50.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:50.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:50.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:07:50.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:50.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.36.2:0
03:07:50.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.14.36.2
03:07:50.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.36.2
03:07:50.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:50.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.14.36.62:35391,127.14.36.61:45213,127.14.36.60:45755
03:07:50.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab
03:07:50.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.36.2
03:07:50.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:50.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:07:50.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:07:50.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:07:50.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.36.20:43983
03:07:50.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:50.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:50.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:50.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:07:51.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:51.014807 14981 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:51.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:51.015385 14981 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:51.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:51.016165 14981 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:51.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:51.048979 14981 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:51.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:51.050047 14981 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.14.36.2
03:07:51.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:51.088575 14981 tablet_server_runner.cc:78] Tablet server non-default flags:
03:07:51.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:51.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.36.20:43983
03:07:51.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:51.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:51.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-1/data
03:07:51.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-1/wal
03:07:51.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab
03:07:51.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.36.2
03:07:51.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:51.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:51.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.36.2:0
03:07:51.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:51.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:51.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:07:51.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:07:51.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:51.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:51.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-1/data/info.pb
03:07:51.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.36.2
03:07:51.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:51.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.14.36.62:35391,127.14.36.61:45213,127.14.36.60:45755
03:07:51.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:51.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:51.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:51.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:51.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:51.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.14.36.2
03:07:51.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-1/logs
03:07:51.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:51.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:51.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:51.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:07:51.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:07:51.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
03:07:51.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:51.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Jan 2026 02:43:21 UTC on 5fd53c4cbb9d
03:07:51.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10241
03:07:51.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:51.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:51.090260 14981 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:51.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:51.091368 14981 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:51.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:51.107259 14988 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:51.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:07:51 dist-test-slave-7fb5 krb5kdc[14493](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656071, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:51.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:51.486443 14529 init.cc:295] Successfully reacquired a new kerberos TGT
03:07:51.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:51.665336 14968 heartbeater.cc:499] Master 127.14.36.61:45213 was elected leader, sending a full tablet report...
03:07:52.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:51.108726 14987 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:52.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:51.112105 14990 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:52.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:52.562253 14989 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1450 milliseconds
03:07:52.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:52.562386 14981 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:07:52.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:52.564353 14981 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:52.571552 14981 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:52.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:52.573376 14981 hybrid_clock.cc:648] HybridClock initialized: now 1769656072573336 us; error 48 us; skew 500 ppm
03:07:52.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:52.574503 14981 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:52.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:07:52 dist-test-slave-7fb5 krb5kdc[14493](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656072, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.2@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:52.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:52.582535 14981 init.cc:377] Logged in from keytab as kudu/127.14.36.2@KRBTEST.COM (short username kudu)
03:07:52.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:52.587735 14981 webserver.cc:492] Webserver started at http://127.14.36.2:33871/ using document root <none> and password file <none>
03:07:52.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:52.588699 14981 fs_manager.cc:362] Metadata directory not provided
03:07:52.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:52.588888 14981 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:52.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:52.589283 14981 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
03:07:52.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:52.593854 14981 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-1/data/instance:
03:07:52.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2bb5a457deff4c0ba8fcedcccfb2b50b"
03:07:52.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-29 03:07:52 on dist-test-slave-7fb5"
03:07:52.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:52.594910 14981 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-1/wal/instance:
03:07:52.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2bb5a457deff4c0ba8fcedcccfb2b50b"
03:07:52.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-29 03:07:52 on dist-test-slave-7fb5"
03:07:52.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:52.602231 14981 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.003s
03:07:52.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:52.607980 15002 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:52.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:52.609138 14981 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s
03:07:52.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:52.609539 14981 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-1/data,/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-1/wal
03:07:52.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2bb5a457deff4c0ba8fcedcccfb2b50b"
03:07:52.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-29 03:07:52 on dist-test-slave-7fb5"
03:07:52.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:52.609894 14981 fs_report.cc:389] FS layout report
03:07:52.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:52.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-1/wal
03:07:52.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-1/wal
03:07:52.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-1/data/data
03:07:52.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:52.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:52.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:52.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:52.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:52.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:52.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:52.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:52.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:52.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:52.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:52.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:52.659153 14981 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:52.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:52.669821 14981 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:52.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:52.671334 14981 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:52.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:52.674467 14981 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
03:07:52.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:52.678833 14981 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:07:52.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:52.679136 14981 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:07:52.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:52.679457 14981 ts_tablet_manager.cc:616] Registered 0 tablets
03:07:52.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:52.679703 14981 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.001s sys 0.000s
03:07:52.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:52.830924 14981 rpc_server.cc:307] RPC server started. Bound to: 127.14.36.2:37651
03:07:52.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:52.831048 15116 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.14.36.2:37651 every 8 connection(s)
03:07:52.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:52.833612 14981 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-1/data/info.pb
03:07:52.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:52.838171 14480 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu as pid 14981
03:07:52.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:52.838760 14480 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-1/wal/instance
03:07:52.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:07:52 dist-test-slave-7fb5 krb5kdc[14493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656072, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.2@KRBTEST.COM for kudu/127.14.36.60@KRBTEST.COM
03:07:52.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.14.36.3@KRBTEST.COM; defaulting to no policy
03:07:52.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:52.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.14.36.3@KRBTEST.COM" created.
03:07:52.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:52.897815 15117 heartbeater.cc:344] Connected to a master server at 127.14.36.60:45755
03:07:52.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:52.898370 15117 heartbeater.cc:461] Registering TS with master...
03:07:52.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:52.899717 15117 heartbeater.cc:507] Master 127.14.36.60:45755 requested a full tablet report, sending...
03:07:52.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:07:52 dist-test-slave-7fb5 krb5kdc[14493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656072, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.2@KRBTEST.COM for kudu/127.14.36.62@KRBTEST.COM
03:07:52.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:52.903493 14723 ts_manager.cc:194] Registered new tserver with Master: 2bb5a457deff4c0ba8fcedcccfb2b50b (127.14.36.2:37651)
03:07:52.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:07:52 dist-test-slave-7fb5 krb5kdc[14493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656072, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.2@KRBTEST.COM for kudu/127.14.36.61@KRBTEST.COM
03:07:52.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:52.919489 15119 heartbeater.cc:344] Connected to a master server at 127.14.36.62:35391
03:07:52.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:52.919869 15119 heartbeater.cc:461] Registering TS with master...
03:07:52.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:52.920934 15119 heartbeater.cc:507] Master 127.14.36.62:35391 requested a full tablet report, sending...
03:07:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:52.923684 14551 ts_manager.cc:194] Registered new tserver with Master: 2bb5a457deff4c0ba8fcedcccfb2b50b (127.14.36.2:37651)
03:07:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.14.36.3 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab.
03:07:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.14.36.3 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab.
03:07:52.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:52.924791 15118 heartbeater.cc:344] Connected to a master server at 127.14.36.61:45213
03:07:52.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:52.925168 15118 heartbeater.cc:461] Registering TS with master...
03:07:52.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:52.926244 15118 heartbeater.cc:507] Master 127.14.36.61:45213 requested a full tablet report, sending...
03:07:52.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:52.928819 14634 ts_manager.cc:194] Registered new tserver with Master: 2bb5a457deff4c0ba8fcedcccfb2b50b (127.14.36.2:37651)
03:07:52.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:52.930651 14634 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.14.36.2@KRBTEST.COM'} at 127.14.36.2:38507
03:07:52.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.14.36.3@KRBTEST.COM; defaulting to no policy
03:07:52.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:52.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.14.36.3@KRBTEST.COM" created.
03:07:52.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:52.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.14.36.3 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab.
03:07:52.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.14.36.3 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab.
03:07:52.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:52.969029 14480 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu
03:07:52.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu
03:07:52.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-2/wal
03:07:52.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-2/data
03:07:52.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:52.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:52.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:52.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:52.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:52.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:52.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-2/logs
03:07:52.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-2/data/info.pb
03:07:52.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:52.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:52.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:52.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:52.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:52.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:52.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:52.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:52.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:07:52.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:52.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.36.3:0
03:07:52.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.14.36.3
03:07:52.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.36.3
03:07:52.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:52.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.14.36.62:35391,127.14.36.61:45213,127.14.36.60:45755
03:07:52.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab
03:07:52.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.36.3
03:07:52.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:52.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:07:52.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:07:52.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:07:52.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.36.20:43983
03:07:52.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:52.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:52.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:52.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:07:53.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:53.282217 15131 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:53.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:53.282677 15131 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:53.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:53.283257 15131 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:53.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:53.316733 15131 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:53.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:53.317657 15131 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.14.36.3
03:07:53.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:53.356053 15131 tablet_server_runner.cc:78] Tablet server non-default flags:
03:07:53.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:53.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.36.20:43983
03:07:53.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:53.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:53.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-2/data
03:07:53.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-2/wal
03:07:53.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab
03:07:53.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.36.3
03:07:53.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:53.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:53.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.36.3:0
03:07:53.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:53.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:53.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:07:53.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:07:53.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:53.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:53.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-2/data/info.pb
03:07:53.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.36.3
03:07:53.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:53.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.14.36.62:35391,127.14.36.61:45213,127.14.36.60:45755
03:07:53.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:53.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:53.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:53.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:53.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:53.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.14.36.3
03:07:53.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-2/logs
03:07:53.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:53.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:53.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:53.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:07:53.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:07:53.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
03:07:53.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:53.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Jan 2026 02:43:21 UTC on 5fd53c4cbb9d
03:07:53.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10241
03:07:53.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:53.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:53.357683 15131 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:53.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:53.358711 15131 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:53.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:53.372825 15137 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:53.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:07:53 dist-test-slave-7fb5 krb5kdc[14493](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656073, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:53.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:53.545140 14613 init.cc:295] Successfully reacquired a new kerberos TGT
03:07:53.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:53.934338 15118 heartbeater.cc:499] Master 127.14.36.61:45213 was elected leader, sending a full tablet report...
03:07:55.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:54.776376 15136 debug-util.cc:398] Leaking SignalData structure 0x7b08000282a0 after lost signal to thread 15131
03:07:55.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:53.373011 15138 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:55.003374 15136 kernel_stack_watchdog.cc:198] Thread 15131 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 401ms:
03:07:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
03:07:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
03:07:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
03:07:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
03:07:55.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:55.004868 15131 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.631s user 0.561s sys 1.056s
03:07:55.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:55.005275 15131 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.633s user 0.561s sys 1.056s
03:07:55.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:55.006340 15140 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:55.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:55.009361 15139 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1630 milliseconds
03:07:55.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.009425 15131 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:07:55.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.010681 15131 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:55.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.012791 15131 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:55.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.014205 15131 hybrid_clock.cc:648] HybridClock initialized: now 1769656075014163 us; error 48 us; skew 500 ppm
03:07:55.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.015172 15131 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:55.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:07:55 dist-test-slave-7fb5 krb5kdc[14493](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656075, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.3@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:55.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.021476 15131 init.cc:377] Logged in from keytab as kudu/127.14.36.3@KRBTEST.COM (short username kudu)
03:07:55.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.026737 15131 webserver.cc:492] Webserver started at http://127.14.36.3:40577/ using document root <none> and password file <none>
03:07:55.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.028062 15131 fs_manager.cc:362] Metadata directory not provided
03:07:55.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.028411 15131 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:55.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.028913 15131 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
03:07:55.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.033239 15131 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-2/data/instance:
03:07:55.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a0016e6f9c82495288bb9911310fe995"
03:07:55.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-29 03:07:55 on dist-test-slave-7fb5"
03:07:55.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.034303 15131 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-2/wal/instance:
03:07:55.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a0016e6f9c82495288bb9911310fe995"
03:07:55.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-29 03:07:55 on dist-test-slave-7fb5"
03:07:55.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.041728 15131 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.008s sys 0.000s
03:07:55.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.047322 15151 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:55.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.048511 15131 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
03:07:55.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.048887 15131 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-2/data,/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-2/wal
03:07:55.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a0016e6f9c82495288bb9911310fe995"
03:07:55.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-29 03:07:55 on dist-test-slave-7fb5"
03:07:55.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.049256 15131 fs_report.cc:389] FS layout report
03:07:55.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:55.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-2/wal
03:07:55.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-2/wal
03:07:55.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-2/data/data
03:07:55.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:55.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:55.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:55.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:55.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:55.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:55.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:55.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:55.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:55.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:55.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:55.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.104454 15131 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:55.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.110894 15131 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:55.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.111320 15131 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:55.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.114176 15131 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
03:07:55.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.118763 15131 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:07:55.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.119020 15131 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:07:55.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.119313 15131 ts_tablet_manager.cc:616] Registered 0 tablets
03:07:55.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.119501 15131 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:07:55.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.281972 15131 rpc_server.cc:307] RPC server started. Bound to: 127.14.36.3:34141
03:07:55.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.282056 15263 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.14.36.3:34141 every 8 connection(s)
03:07:55.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.285154 15131 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-2/data/info.pb
03:07:55.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.293107 14480 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu as pid 15131
03:07:55.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.293752 14480 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/ts-2/wal/instance
03:07:55.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:07:55 dist-test-slave-7fb5 krb5kdc[14493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656075, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.3@KRBTEST.COM for kudu/127.14.36.60@KRBTEST.COM
03:07:55.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:07:55 dist-test-slave-7fb5 krb5kdc[14493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656075, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.3@KRBTEST.COM for kudu/127.14.36.62@KRBTEST.COM
03:07:55.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:07:55 dist-test-slave-7fb5 krb5kdc[14493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656075, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.3@KRBTEST.COM for kudu/127.14.36.61@KRBTEST.COM
03:07:55.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.350329 15266 heartbeater.cc:344] Connected to a master server at 127.14.36.62:35391
03:07:55.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.350548 15264 heartbeater.cc:344] Connected to a master server at 127.14.36.60:45755
03:07:55.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.350873 15266 heartbeater.cc:461] Registering TS with master...
03:07:55.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.350939 15264 heartbeater.cc:461] Registering TS with master...
03:07:55.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.352458 15266 heartbeater.cc:507] Master 127.14.36.62:35391 requested a full tablet report, sending...
03:07:55.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.354018 15264 heartbeater.cc:507] Master 127.14.36.60:45755 requested a full tablet report, sending...
03:07:55.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.355954 14551 ts_manager.cc:194] Registered new tserver with Master: a0016e6f9c82495288bb9911310fe995 (127.14.36.3:34141)
03:07:55.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.357775 14723 ts_manager.cc:194] Registered new tserver with Master: a0016e6f9c82495288bb9911310fe995 (127.14.36.3:34141)
03:07:55.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.358673 15265 heartbeater.cc:344] Connected to a master server at 127.14.36.61:45213
03:07:55.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.359037 15265 heartbeater.cc:461] Registering TS with master...
03:07:55.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.360033 15265 heartbeater.cc:507] Master 127.14.36.61:45213 requested a full tablet report, sending...
03:07:55.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.362430 14634 ts_manager.cc:194] Registered new tserver with Master: a0016e6f9c82495288bb9911310fe995 (127.14.36.3:34141)
03:07:55.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.364176 14634 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.14.36.3@KRBTEST.COM'} at 127.14.36.3:49155
03:07:55.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.378978 14480 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
03:07:55.390 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:55.392 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
03:07:55.402 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
03:07:55.403 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
03:07:55.406 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.14.36.62"
port: 35391
}
bound_http_address {
host: "127.14.36.62"
port: 37233
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.14.36.61"
port: 45213
}
bound_http_address {
host: "127.14.36.61"
port: 44571
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.14.36.60"
port: 45755
}
bound_http_address {
host: "127.14.36.60"
port: 41699
}
}
}
03:07:55.406 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
03:07:55.408 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.14.36.1"
port: 45385
}
bound_http_address {
host: "127.14.36.1"
port: 41393
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.14.36.2"
port: 37651
}
bound_http_address {
host: "127.14.36.2"
port: 33871
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.14.36.3"
port: 34141
}
bound_http_address {
host: "127.14.36.3"
port: 40577
}
}
}
03:07:55.409 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
03:07:55.410 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
03:07:55.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:07:55 dist-test-slave-7fb5 krb5kdc[14493](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656075, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:55.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
03:07:55.441 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:55.443 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/krb5cc
03:07:55.453 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:07:55.454 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/krb5cc
03:07:55.456 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:07:55.459 [DEBUG - Time-limited test] (SecurityContext.java:158) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
03:07:55.461 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
03:07:55.461 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
03:07:55.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:07:55 dist-test-slave-7fb5 krb5kdc[14493](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656075, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:55.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
03:07:55.492 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:55.493 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/krb5cc
03:07:55.494 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:07:55.495 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.14.36.62:35391
03:07:55.496 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
03:07:55.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.497298 14480 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu with pid 14513
03:07:55.526 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:55.527 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.14.36.61:45213
03:07:55.527 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
03:07:55.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.527761 14480 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu with pid 14596
03:07:55.559 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:55.559 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.14.36.60:45755
03:07:55.559 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
03:07:55.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.559820 14480 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu with pid 14679
03:07:55.586 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:55.586 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.14.36.62:35391
03:07:55.591 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
03:07:55.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.593361 14480 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu
03:07:55.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu
03:07:55.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/wal
03:07:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/data
03:07:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/logs
03:07:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/data/info.pb
03:07:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:07:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.36.62:35391
03:07:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.36.62
03:07:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37233
03:07:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab
03:07:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.36.62
03:07:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:07:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:07:55.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:07:55.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.36.20:43983
03:07:55.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:55.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:55.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:55.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:55.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.36.62:35391,127.14.36.61:45213,127.14.36.60:45755
03:07:55.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:55.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37645/jwks.json
03:07:55.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:07:55.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:07:55.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:55.691391 14969 heartbeater.cc:646] Failed to heartbeat to 127.14.36.62:35391 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.14.36.62:35391: connect: Connection refused (error 111)
03:07:55.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:55.917357 15278 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:55.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:55.918226 15278 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:55.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:55.919018 15278 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:55.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:55.919376 15117 heartbeater.cc:646] Failed to heartbeat to 127.14.36.60:45755 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.14.36.60:45755: connect: Connection refused (error 111)
03:07:55.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:55.954882 15278 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:07:55.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:55.955188 15278 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:07:55.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:55.955570 15278 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:55.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:55.956072 15278 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:07:55.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:55.956445 15278 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:07:55.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:55.956753 15278 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:07:55.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:55.957028 15278 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37645/jwks.json
03:07:55.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.996655 15278 master_runner.cc:386] Master server non-default flags:
03:07:55.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:55.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.36.20:43983
03:07:55.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:55.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:55.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/data
03:07:55.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/wal
03:07:55.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:07:55.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:55.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.36.62:35391,127.14.36.61:45213,127.14.36.60:45755
03:07:55.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab
03:07:55.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.36.62
03:07:55.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:55.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:55.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:55.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.36.62:35391
03:07:55.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:55.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:55.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:55.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37645/jwks.json
03:07:55.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:55.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:07:55.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:07:55.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:55.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:55.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/data/info.pb
03:07:55.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.36.62
03:07:55.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37233
03:07:55.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:55.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:55.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:55.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:55.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:55.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/logs
03:07:55.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:55.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:55.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:55.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:07:55.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:07:55.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
03:07:55.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:55.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Jan 2026 02:43:21 UTC on 5fd53c4cbb9d
03:07:55.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10241
03:07:55.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:55.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.998361 15278 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:55.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:55.999665 15278 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:56.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:56.010739 15289 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:56.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:56.367719 15265 heartbeater.cc:499] Master 127.14.36.61:45213 was elected leader, sending a full tablet report...
03:07:56.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:56.380245 15266 heartbeater.cc:646] Failed to heartbeat to 127.14.36.62:35391 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.14.36.62:35391: connect: Connection refused (error 111)
03:07:57.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:56.016367 15292 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:57.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:56.012593 15290 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:57.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:57.451488 15291 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1436 milliseconds
03:07:57.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:57.451611 15278 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:07:57.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:57.453171 15278 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:57.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:57.455722 15278 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:57.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:57.457152 15278 hybrid_clock.cc:648] HybridClock initialized: now 1769656077457077 us; error 92 us; skew 500 ppm
03:07:57.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:57.458243 15278 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:57.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:07:57 dist-test-slave-7fb5 krb5kdc[14493](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656077, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:57.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:57.464814 15278 init.cc:377] Logged in from keytab as kudu/127.14.36.62@KRBTEST.COM (short username kudu)
03:07:57.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:57.471524 15278 webserver.cc:492] Webserver started at http://127.14.36.62:37233/ using document root <none> and password file <none>
03:07:57.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:57.472497 15278 fs_manager.cc:362] Metadata directory not provided
03:07:57.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:57.472707 15278 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:57.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:57.481074 15278 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.008s sys 0.000s
03:07:57.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:57.485888 15309 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:57.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:57.487087 15278 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
03:07:57.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:57.487387 15278 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/data,/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/wal
03:07:57.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6cad901ab52a4c7b9a90110bcabd529d"
03:07:57.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-29 03:07:41 on dist-test-slave-7fb5"
03:07:57.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:57.489452 15278 fs_report.cc:389] FS layout report
03:07:57.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:57.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/wal
03:07:57.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/wal
03:07:57.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/data/data
03:07:57.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:57.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:57.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:57.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:57.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:57.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:57.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:57.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:57.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:57.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:57.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:57.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:57.542878 15278 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:57.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:57.549111 15278 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:57.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:57.549520 15278 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:57.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:57.629964 15278 rpc_server.cc:307] RPC server started. Bound to: 127.14.36.62:35391
03:07:57.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:57.630066 15361 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.14.36.62:35391 every 8 connection(s)
03:07:57.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:57.632664 15278 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/data/info.pb
03:07:57.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:57.637176 14480 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu as pid 15278
03:07:57.638 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:57.638 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.14.36.61:45213
03:07:57.638 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
03:07:57.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:57.640339 14480 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu
03:07:57.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu
03:07:57.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/wal
03:07:57.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/data
03:07:57.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/logs
03:07:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/data/info.pb
03:07:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:07:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.36.61:45213
03:07:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.36.61
03:07:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44571
03:07:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab
03:07:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.36.61
03:07:57.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:57.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:07:57.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:07:57.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:07:57.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.36.20:43983
03:07:57.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:57.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:57.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:57.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:57.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.36.62:35391,127.14.36.61:45213,127.14.36.60:45755
03:07:57.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:57.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37645/jwks.json
03:07:57.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:07:57.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:07:57.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:57.645012 15362 sys_catalog.cc:263] Verifying existing consensus state
03:07:57.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:57.654585 15362 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d: Bootstrap starting.
03:07:57.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:57.709108 15362 log.cc:826] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d: Log is configured to *not* fsync() on all Append() calls
03:07:57.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:57.745244 15362 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:07:57.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:57.746171 15362 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d: Bootstrap complete.
03:07:57.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:57.765352 14969 heartbeater.cc:344] Connected to a master server at 127.14.36.62:35391
03:07:57.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:57.772651 15362 raft_consensus.cc:359] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } }
03:07:57.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:57.775158 15362 raft_consensus.cc:740] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6cad901ab52a4c7b9a90110bcabd529d, State: Initialized, Role: FOLLOWER
03:07:57.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:57.775969 15362 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } }
03:07:57.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:57.778861 15369 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } } }
03:07:57.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:57.779743 15369 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [sys.catalog]: This master's current role is: FOLLOWER
03:07:57.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:57.780251 15362 sys_catalog.cc:565] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [sys.catalog]: configured and running, proceeding with master startup.
03:07:57.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:57.826890 15380 catalog_manager.cc:1269] Loaded cluster ID: 5d811d19a7434727a35b22e3a8e4d9bd
03:07:57.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:57.827454 15380 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d: loading cluster ID for follower catalog manager: success
03:07:57.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:07:57 dist-test-slave-7fb5 krb5kdc[14493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656077, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.62@KRBTEST.COM for kudu/127.14.36.62@KRBTEST.COM
03:07:57.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:57.862679 15380 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d: acquiring CA information for follower catalog manager: success
03:07:57.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:57.869791 15380 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:07:57.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:57.865732 15278 master_runner.cc:186] Error getting master registration for 127.14.36.61:45213: Network error: Client connection negotiation failed: client connection to 127.14.36.61:45213: connect: Connection refused (error 111),
03:07:57.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:57.893265 15278 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:07:57.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:57.952327 15119 heartbeater.cc:344] Connected to a master server at 127.14.36.62:35391
03:07:57.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:57.958788 15326 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "2bb5a457deff4c0ba8fcedcccfb2b50b" instance_seqno: 1769656072795400) as {username='kudu', principal='kudu/127.14.36.2@KRBTEST.COM'} at 127.14.36.2:47725; Asking this server to re-register.
03:07:57.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:57.961272 15119 heartbeater.cc:461] Registering TS with master...
03:07:57.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:57.965605 15326 ts_manager.cc:194] Registered new tserver with Master: 2bb5a457deff4c0ba8fcedcccfb2b50b (127.14.36.2:37651)
03:07:58.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:58.048238 15364 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:58.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:58.048919 15364 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:58.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:58.049787 15364 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:58.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:58.084314 15364 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:07:58.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:58.084619 15364 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:07:58.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:58.085032 15364 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:58.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:58.085386 15364 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:07:58.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:58.085704 15364 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:07:58.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:58.086021 15364 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:07:58.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:58.086285 15364 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37645/jwks.json
03:07:58.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:58.126710 15364 master_runner.cc:386] Master server non-default flags:
03:07:58.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:58.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.36.20:43983
03:07:58.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:58.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:58.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/data
03:07:58.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/wal
03:07:58.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:07:58.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:58.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.36.62:35391,127.14.36.61:45213,127.14.36.60:45755
03:07:58.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab
03:07:58.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.36.61
03:07:58.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:58.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:58.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:58.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.36.61:45213
03:07:58.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:58.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:58.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:58.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37645/jwks.json
03:07:58.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:58.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:07:58.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:07:58.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:58.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:58.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/data/info.pb
03:07:58.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.36.61
03:07:58.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44571
03:07:58.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:58.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:58.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:58.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:58.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:58.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/logs
03:07:58.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:58.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:58.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:58.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:07:58.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:07:58.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
03:07:58.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:58.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Jan 2026 02:43:21 UTC on 5fd53c4cbb9d
03:07:58.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10241
03:07:58.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:58.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:58.128376 15364 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:58.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:58.129777 15364 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:58.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:58.145337 15390 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:58.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:58.424834 15266 heartbeater.cc:344] Connected to a master server at 127.14.36.62:35391
03:07:58.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:58.429131 15326 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "a0016e6f9c82495288bb9911310fe995" instance_seqno: 1769656075243095) as {username='kudu', principal='kudu/127.14.36.3@KRBTEST.COM'} at 127.14.36.3:42087; Asking this server to re-register.
03:07:58.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:58.430757 15266 heartbeater.cc:461] Registering TS with master...
03:07:58.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:58.434597 15326 ts_manager.cc:194] Registered new tserver with Master: a0016e6f9c82495288bb9911310fe995 (127.14.36.3:34141)
03:07:58.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:58.771669 15326 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "1e74ae69837f4a19929b3bf70017a23d" instance_seqno: 1769656070527044) as {username='kudu', principal='kudu/127.14.36.1@KRBTEST.COM'} at 127.14.36.1:50081; Asking this server to re-register.
03:07:58.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:58.773629 14969 heartbeater.cc:461] Registering TS with master...
03:07:58.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:58.777047 15326 ts_manager.cc:194] Registered new tserver with Master: 1e74ae69837f4a19929b3bf70017a23d (127.14.36.1:45385)
03:07:59.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:59.286960 15401 raft_consensus.cc:493] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:07:59.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:59.288170 15401 raft_consensus.cc:515] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } }
03:07:59.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:59.296096 15401 leader_election.cc:290] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers de1dd59a869a46bdb0165c8963557689 (127.14.36.61:45213), a5feff59896e4438ac3b32435885f30c (127.14.36.60:45755)
03:07:59.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:59.310628 15313 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.14.36.61:45213: connect: Connection refused (error 111)
03:07:59.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:59.322889 15313 leader_election.cc:336] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer de1dd59a869a46bdb0165c8963557689 (127.14.36.61:45213): Network error: Client connection negotiation failed: client connection to 127.14.36.61:45213: connect: Connection refused (error 111)
03:07:59.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:59.326339 15313 leader_election.cc:336] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer a5feff59896e4438ac3b32435885f30c (127.14.36.60:45755): Network error: Client connection negotiation failed: client connection to 127.14.36.60:45755: connect: Connection refused (error 111)
03:07:59.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:59.327216 15313 leader_election.cc:304] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6cad901ab52a4c7b9a90110bcabd529d; no voters: a5feff59896e4438ac3b32435885f30c, de1dd59a869a46bdb0165c8963557689
03:07:59.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:59.328510 15401 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
03:07:59.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:58.146502 15391 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:59.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:59.545665 15389 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 15364
03:07:59.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:59.945354 15364 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.802s user 0.669s sys 1.071s
03:07:59.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:59.946693 15364 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.804s user 0.669s sys 1.071s
03:07:59.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:07:59.961748 15408 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:59.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:59.961983 15364 server_base.cc:1060] running on GCE node
03:07:59.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:59.963238 15364 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:59.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:59.965878 15364 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:59.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:59.967311 15364 hybrid_clock.cc:648] HybridClock initialized: now 1769656079967236 us; error 56 us; skew 500 ppm
03:07:59.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:59.968533 15364 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:59.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:07:59 dist-test-slave-7fb5 krb5kdc[14493](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656079, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:59.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:59.975209 15364 init.cc:377] Logged in from keytab as kudu/127.14.36.61@KRBTEST.COM (short username kudu)
03:07:59.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:59.985283 15364 webserver.cc:492] Webserver started at http://127.14.36.61:44571/ using document root <none> and password file <none>
03:07:59.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:59.986449 15364 fs_manager.cc:362] Metadata directory not provided
03:07:59.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:59.986755 15364 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:59.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:07:59.995471 15364 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.006s sys 0.000s
03:08:00.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:00.000945 15415 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:08:00.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:00.002326 15364 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.004s sys 0.000s
03:08:00.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:00.002624 15364 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/data,/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/wal
03:08:00.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "de1dd59a869a46bdb0165c8963557689"
03:08:00.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-29 03:07:43 on dist-test-slave-7fb5"
03:08:00.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:00.004567 15364 fs_report.cc:389] FS layout report
03:08:00.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:08:00.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/wal
03:08:00.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/wal
03:08:00.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/data/data
03:08:00.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:08:00.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:08:00.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:08:00.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:08:00.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:08:00.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:08:00.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:08:00.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:08:00.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:08:00.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:08:00.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:08:00.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:00.056207 15364 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:08:00.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:00.061955 15364 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:08:00.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:00.062505 15364 kserver.cc:163] Server-wide thread pool size limit: 3276
03:08:00.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:00.151412 15364 rpc_server.cc:307] RPC server started. Bound to: 127.14.36.61:45213
03:08:00.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:00.151531 15466 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.14.36.61:45213 every 8 connection(s)
03:08:00.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:00.154227 15364 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/data/info.pb
03:08:00.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:00.155158 14480 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu as pid 15364
03:08:00.156 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:08:00.156 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.14.36.60:45755
03:08:00.157 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
03:08:00.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:00.159842 14480 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu
03:08:00.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu
03:08:00.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/wal
03:08:00.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/data
03:08:00.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:08:00.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:08:00.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:08:00.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:00.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:00.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:00.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/logs
03:08:00.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/data/info.pb
03:08:00.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:00.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:08:00.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:08:00.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:08:00.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:08:00.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:00.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:00.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:00.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:08:00.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:08:00.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:00.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:00.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.36.60:45755
03:08:00.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.36.60
03:08:00.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41699
03:08:00.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab
03:08:00.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.36.60
03:08:00.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:08:00.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:08:00.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:08:00.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:08:00.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.36.20:43983
03:08:00.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:00.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:00.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:00.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:00.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.36.62:35391,127.14.36.61:45213,127.14.36.60:45755
03:08:00.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:00.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37645/jwks.json
03:08:00.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:08:00.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:08:00.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:00.172015 15467 sys_catalog.cc:263] Verifying existing consensus state
03:08:00.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:00.177946 15467 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689: Bootstrap starting.
03:08:00.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:00.235050 15467 log.cc:826] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689: Log is configured to *not* fsync() on all Append() calls
03:08:00.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:08:00 dist-test-slave-7fb5 krb5kdc[14493](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656080, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.1@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:00.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:00.241963 14848 init.cc:295] Successfully reacquired a new kerberos TGT
03:08:00.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:00.270373 15467 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:08:00.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:00.271384 15467 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689: Bootstrap complete.
03:08:00.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:00.306396 15467 raft_consensus.cc:359] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } }
03:08:00.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:00.309929 15467 raft_consensus.cc:740] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: de1dd59a869a46bdb0165c8963557689, State: Initialized, Role: FOLLOWER
03:08:00.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:00.311910 15467 consensus_queue.cc:260] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } }
03:08:00.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:00.315722 15471 sys_catalog.cc:455] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } } }
03:08:00.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:00.316879 15471 sys_catalog.cc:458] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [sys.catalog]: This master's current role is: FOLLOWER
03:08:00.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:00.317090 15467 sys_catalog.cc:565] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [sys.catalog]: configured and running, proceeding with master startup.
03:08:00.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:00.400956 15482 catalog_manager.cc:1269] Loaded cluster ID: 5d811d19a7434727a35b22e3a8e4d9bd
03:08:00.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:00.401224 15482 catalog_manager.cc:1562] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689: loading cluster ID for follower catalog manager: success
03:08:00.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:00.416692 15482 catalog_manager.cc:1584] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689: acquiring CA information for follower catalog manager: success
03:08:00.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:08:00 dist-test-slave-7fb5 krb5kdc[14493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656079, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.61@KRBTEST.COM for kudu/127.14.36.62@KRBTEST.COM
03:08:00.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:00.432301 15482 catalog_manager.cc:1612] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:08:00.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:00.514072 15265 heartbeater.cc:344] Connected to a master server at 127.14.36.61:45213
03:08:00.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:00.514600 15265 heartbeater.cc:499] Master 127.14.36.61:45213 was elected leader, sending a full tablet report...
03:08:00.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:00.537019 15432 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "a0016e6f9c82495288bb9911310fe995" instance_seqno: 1769656075243095) as {username='kudu', principal='kudu/127.14.36.3@KRBTEST.COM'} at 127.14.36.3:34259; Asking this server to re-register.
03:08:00.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:00.545574 15265 heartbeater.cc:461] Registering TS with master...
03:08:00.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:00.552834 15432 ts_manager.cc:194] Registered new tserver with Master: a0016e6f9c82495288bb9911310fe995 (127.14.36.3:34141)
03:08:00.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:00.501042 15364 master_runner.cc:186] Error getting master registration for 127.14.36.60:45755: Network error: Client connection negotiation failed: client connection to 127.14.36.60:45755: connect: Connection refused (error 111),
03:08:00.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:00.556950 15364 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:08:00.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:00.672474 15469 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:08:00.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:00.673295 15469 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:08:00.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:00.674422 15469 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:08:00.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:00.717095 15469 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:08:00.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:00.717393 15469 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:08:00.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:00.717788 15469 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:08:00.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:00.718125 15469 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:08:00.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:00.718502 15469 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:08:00.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:00.718838 15469 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:08:00.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:00.719159 15469 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37645/jwks.json
03:08:00.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:00.762071 15469 master_runner.cc:386] Master server non-default flags:
03:08:00.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:00.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.36.20:43983
03:08:00.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:00.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:00.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/data
03:08:00.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/wal
03:08:00.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:08:00.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:00.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.36.62:35391,127.14.36.61:45213,127.14.36.60:45755
03:08:00.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab
03:08:00.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.36.60
03:08:00.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:00.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:00.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:00.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.36.60:45755
03:08:00.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:00.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:08:00.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:00.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37645/jwks.json
03:08:00.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:08:00.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:08:00.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:08:00.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:00.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:00.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/data/info.pb
03:08:00.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.36.60
03:08:00.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41699
03:08:00.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:08:00.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:00.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:08:00.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:08:00.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:00.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/logs
03:08:00.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:00.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:08:00.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:08:00.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:08:00.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:08:00.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
03:08:00.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:08:00.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Jan 2026 02:43:21 UTC on 5fd53c4cbb9d
03:08:00.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10241
03:08:00.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:08:00.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:00.763778 15469 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:08:00.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:00.765231 15469 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:08:00.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:00.779752 15495 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:00.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:00.832794 14968 heartbeater.cc:344] Connected to a master server at 127.14.36.61:45213
03:08:00.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:00.841915 15432 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "1e74ae69837f4a19929b3bf70017a23d" instance_seqno: 1769656070527044) as {username='kudu', principal='kudu/127.14.36.1@KRBTEST.COM'} at 127.14.36.1:50923; Asking this server to re-register.
03:08:00.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:00.844415 14968 heartbeater.cc:461] Registering TS with master...
03:08:00.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:00.847635 15432 ts_manager.cc:194] Registered new tserver with Master: 1e74ae69837f4a19929b3bf70017a23d (127.14.36.1:45385)
03:08:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:01.044317 15118 heartbeater.cc:344] Connected to a master server at 127.14.36.61:45213
03:08:01.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:01.048588 15432 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "2bb5a457deff4c0ba8fcedcccfb2b50b" instance_seqno: 1769656072795400) as {username='kudu', principal='kudu/127.14.36.2@KRBTEST.COM'} at 127.14.36.2:47957; Asking this server to re-register.
03:08:01.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:01.051271 15118 heartbeater.cc:461] Registering TS with master...
03:08:01.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:01.056224 15432 ts_manager.cc:194] Registered new tserver with Master: 2bb5a457deff4c0ba8fcedcccfb2b50b (127.14.36.2:37651)
03:08:01.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:01.409793 15505 raft_consensus.cc:493] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:08:01.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:01.410679 15505 raft_consensus.cc:515] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } }
03:08:01.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:01.425547 15505 leader_election.cc:290] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers de1dd59a869a46bdb0165c8963557689 (127.14.36.61:45213), a5feff59896e4438ac3b32435885f30c (127.14.36.60:45755)
03:08:01.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:01.462630 15442 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6cad901ab52a4c7b9a90110bcabd529d" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "de1dd59a869a46bdb0165c8963557689" is_pre_election: true
03:08:01.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:01.463369 15313 leader_election.cc:336] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer a5feff59896e4438ac3b32435885f30c (127.14.36.60:45755): Network error: Client connection negotiation failed: client connection to 127.14.36.60:45755: connect: Connection refused (error 111)
03:08:01.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:01.463582 15442 raft_consensus.cc:2468] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6cad901ab52a4c7b9a90110bcabd529d in term 1.
03:08:01.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:01.466976 15313 leader_election.cc:304] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6cad901ab52a4c7b9a90110bcabd529d, de1dd59a869a46bdb0165c8963557689; no voters: a5feff59896e4438ac3b32435885f30c
03:08:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:01.467967 15505 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [term 1 FOLLOWER]: Leader pre-election won for term 2
03:08:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:01.468385 15505 raft_consensus.cc:493] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:08:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:01.468777 15505 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [term 1 FOLLOWER]: Advancing to term 2
03:08:01.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:01.481078 15505 raft_consensus.cc:515] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } }
03:08:01.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:01.485121 15442 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6cad901ab52a4c7b9a90110bcabd529d" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "de1dd59a869a46bdb0165c8963557689"
03:08:01.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:01.485682 15442 raft_consensus.cc:3060] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [term 1 FOLLOWER]: Advancing to term 2
03:08:01.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:01.495252 15442 raft_consensus.cc:2468] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6cad901ab52a4c7b9a90110bcabd529d in term 2.
03:08:01.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:01.496836 15313 leader_election.cc:304] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6cad901ab52a4c7b9a90110bcabd529d, de1dd59a869a46bdb0165c8963557689; no voters:
03:08:01.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:01.501695 15510 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [term 2 FOLLOWER]: Leader election won for term 2
03:08:01.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:01.501848 15505 leader_election.cc:290] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [CANDIDATE]: Term 2 election: Requested vote from peers de1dd59a869a46bdb0165c8963557689 (127.14.36.61:45213), a5feff59896e4438ac3b32435885f30c (127.14.36.60:45755)
03:08:01.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:01.509569 15313 leader_election.cc:336] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer a5feff59896e4438ac3b32435885f30c (127.14.36.60:45755): Network error: Client connection negotiation failed: client connection to 127.14.36.60:45755: connect: Connection refused (error 111)
03:08:01.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:01.512580 15510 raft_consensus.cc:697] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [term 2 LEADER]: Becoming Leader. State: Replica: 6cad901ab52a4c7b9a90110bcabd529d, State: Running, Role: LEADER
03:08:01.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:01.513873 15510 consensus_queue.cc:237] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } }
03:08:01.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:01.536128 15505 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6cad901ab52a4c7b9a90110bcabd529d. Latest consensus state: current_term: 2 leader_uuid: "6cad901ab52a4c7b9a90110bcabd529d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } } }
03:08:01.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:01.537158 15505 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [sys.catalog]: This master's current role is: LEADER
03:08:01.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:01.558550 15513 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
03:08:01.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:01.573087 15513 catalog_manager.cc:1494] Initializing Kudu cluster ID...
03:08:01.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:01.582592 15513 catalog_manager.cc:1269] Loaded cluster ID: 5d811d19a7434727a35b22e3a8e4d9bd
03:08:01.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:01.583983 15513 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
03:08:01.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:01.589329 15513 catalog_manager.cc:1514] Loading token signing keys...
03:08:01.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:01.601715 15513 catalog_manager.cc:6038] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d: Loaded TSK: 0
03:08:01.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:01.605064 15513 catalog_manager.cc:1524] Initializing in-progress tserver states...
03:08:02.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:02.052496 15442 raft_consensus.cc:1275] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [term 2 FOLLOWER]: Refusing update from remote peer 6cad901ab52a4c7b9a90110bcabd529d: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
03:08:02.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:02.054136 15313 consensus_peers.cc:597] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d -> Peer a5feff59896e4438ac3b32435885f30c (127.14.36.60:45755): Couldn't send request to peer a5feff59896e4438ac3b32435885f30c. Status: Network error: Client connection negotiation failed: client connection to 127.14.36.60:45755: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
03:08:02.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:02.055311 15505 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [LEADER]: Connected to new peer: Peer: permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
03:08:02.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:02.093730 15520 sys_catalog.cc:455] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6cad901ab52a4c7b9a90110bcabd529d. Latest consensus state: current_term: 2 leader_uuid: "6cad901ab52a4c7b9a90110bcabd529d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } } }
03:08:02.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:02.094852 15520 sys_catalog.cc:458] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [sys.catalog]: This master's current role is: FOLLOWER
03:08:02.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:02.120510 15520 sys_catalog.cc:455] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "6cad901ab52a4c7b9a90110bcabd529d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } } }
03:08:02.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:02.121485 15520 sys_catalog.cc:458] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [sys.catalog]: This master's current role is: FOLLOWER
03:08:02.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:02.125946 15505 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "6cad901ab52a4c7b9a90110bcabd529d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } } }
03:08:02.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:02.127027 15505 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [sys.catalog]: This master's current role is: LEADER
03:08:02.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:08:02 dist-test-slave-7fb5 krb5kdc[14493](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656082, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.2@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:02.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:02.587802 14998 init.cc:295] Successfully reacquired a new kerberos TGT
03:08:02.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:02.792663 14969 heartbeater.cc:499] Master 127.14.36.62:35391 was elected leader, sending a full tablet report...
03:08:02.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:02.181282 15493 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 15469
03:08:02.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:02.897878 15493 kernel_stack_watchdog.cc:198] Thread 15469 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 398ms:
03:08:02.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
03:08:02.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
03:08:02.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
03:08:02.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
03:08:02.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:02.901423 15497 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:02.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:02.906175 15469 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.126s user 0.588s sys 1.014s
03:08:02.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:02.906577 15469 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.127s user 0.588s sys 1.014s
03:08:02.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:00.780508 15494 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:02.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:02.912283 15278 master_runner.cc:186] Error getting master registration for 127.14.36.60:45755: Network error: Client connection negotiation failed: client connection to 127.14.36.60:45755: connect: Connection refused (error 111),
03:08:02.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:02.912725 15278 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:08:02.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:02.913509 15496 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2131 milliseconds
03:08:02.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:02.913517 15469 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:08:02.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:02.914958 15469 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:08:02.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:02.918293 15469 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:08:02.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:02.919744 15469 hybrid_clock.cc:648] HybridClock initialized: now 1769656082919689 us; error 67 us; skew 500 ppm
03:08:02.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:02.920688 15469 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:08:02.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:08:02 dist-test-slave-7fb5 krb5kdc[14493](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656082, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:02.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:02.927206 15469 init.cc:377] Logged in from keytab as kudu/127.14.36.60@KRBTEST.COM (short username kudu)
03:08:02.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:02.932848 15469 webserver.cc:492] Webserver started at http://127.14.36.60:41699/ using document root <none> and password file <none>
03:08:02.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:02.933853 15469 fs_manager.cc:362] Metadata directory not provided
03:08:02.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:02.934186 15469 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:08:02.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:02.943274 15469 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.006s sys 0.000s
03:08:02.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:02.948721 15534 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:08:02.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:02.949997 15469 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.000s sys 0.004s
03:08:02.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:02.950311 15469 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/data,/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/wal
03:08:02.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a5feff59896e4438ac3b32435885f30c"
03:08:02.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-29 03:07:45 on dist-test-slave-7fb5"
03:08:02.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:02.952584 15469 fs_report.cc:389] FS layout report
03:08:02.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:08:02.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/wal
03:08:02.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/wal
03:08:02.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/data/data
03:08:02.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:08:02.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:08:02.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:08:02.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:08:02.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:08:02.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:08:02.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:08:02.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:08:02.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:08:02.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:08:02.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:08:02.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:02.986593 15119 heartbeater.cc:499] Master 127.14.36.62:35391 was elected leader, sending a full tablet report...
03:08:03.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:03.011102 15469 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:08:03.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:03.021791 15469 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:08:03.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:03.022344 15469 kserver.cc:163] Server-wide thread pool size limit: 3276
03:08:03.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:03.112633 15469 rpc_server.cc:307] RPC server started. Bound to: 127.14.36.60:45755
03:08:03.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:03.112810 15586 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.14.36.60:45755 every 8 connection(s)
03:08:03.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:03.115965 15469 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/data/info.pb
03:08:03.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:03.124982 14480 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu as pid 15469
03:08:03.126 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:08:03.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:03.130818 15587 sys_catalog.cc:263] Verifying existing consensus state
03:08:03.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:03.142009 15587 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c: Bootstrap starting.
03:08:03.213 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.14.36.62:35391(127.14.36.62:35391)] connecting to peer
03:08:03.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:03.278419 15587 log.cc:826] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c: Log is configured to *not* fsync() on all Append() calls
03:08:03.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:03.310559 15587 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:08:03.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:03.312049 15587 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c: Bootstrap complete.
03:08:03.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:03.360666 15587 raft_consensus.cc:359] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } }
03:08:03.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:03.364159 15587 raft_consensus.cc:740] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: a5feff59896e4438ac3b32435885f30c, State: Initialized, Role: FOLLOWER
03:08:03.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:03.365445 15587 consensus_queue.cc:260] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } }
03:08:03.371 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.14.36.61:45213(127.14.36.61:45213)] connecting to peer
03:08:03.379 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.14.36.60:45755(127.14.36.60:45755)] connecting to peer
03:08:03.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:03.385041 15593 sys_catalog.cc:455] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } } }
03:08:03.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:03.386188 15593 sys_catalog.cc:458] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c [sys.catalog]: This master's current role is: FOLLOWER
03:08:03.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:03.387003 15587 sys_catalog.cc:565] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c [sys.catalog]: configured and running, proceeding with master startup.
03:08:03.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:03.457100 15266 heartbeater.cc:499] Master 127.14.36.62:35391 was elected leader, sending a full tablet report...
03:08:03.463 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.14.36.61:45213(127.14.36.61:45213)] Successfully connected to peer
03:08:03.471 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.14.36.62:35391(127.14.36.62:35391)] Successfully connected to peer
03:08:03.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:08:03 dist-test-slave-7fb5 krb5kdc[14493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656077, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.62@KRBTEST.COM for kudu/127.14.36.60@KRBTEST.COM
03:08:03.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:03.507395 15607 catalog_manager.cc:1269] Loaded cluster ID: 5d811d19a7434727a35b22e3a8e4d9bd
03:08:03.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:03.507647 15607 catalog_manager.cc:1562] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c: loading cluster ID for follower catalog manager: success
03:08:03.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:03.515065 15607 catalog_manager.cc:1584] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c: acquiring CA information for follower catalog manager: success
03:08:03.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:03.529186 15607 catalog_manager.cc:1612] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:08:03.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:03.531563 15562 raft_consensus.cc:3060] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c [term 1 FOLLOWER]: Advancing to term 2
03:08:03.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:08:03 dist-test-slave-7fb5 krb5kdc[14493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656082, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.60@KRBTEST.COM for kudu/127.14.36.62@KRBTEST.COM
03:08:03.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:03.580785 15264 heartbeater.cc:344] Connected to a master server at 127.14.36.60:45755
03:08:03.639 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.14.36.60:45755(127.14.36.60:45755)] Successfully connected to peer
03:08:03.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:03.670789 15618 sys_catalog.cc:455] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "6cad901ab52a4c7b9a90110bcabd529d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } } }
03:08:03.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:03.671766 15618 sys_catalog.cc:458] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c [sys.catalog]: This master's current role is: FOLLOWER
03:08:03.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:03.679924 15552 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "a0016e6f9c82495288bb9911310fe995" instance_seqno: 1769656075243095) as {username='kudu', principal='kudu/127.14.36.3@KRBTEST.COM'} at 127.14.36.3:57131; Asking this server to re-register.
03:08:03.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:03.682235 15264 heartbeater.cc:461] Registering TS with master...
03:08:03.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:03.685761 15551 ts_manager.cc:194] Registered new tserver with Master: a0016e6f9c82495288bb9911310fe995 (127.14.36.3:34141)
03:08:03.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:03.687341 15593 sys_catalog.cc:455] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6cad901ab52a4c7b9a90110bcabd529d. Latest consensus state: current_term: 2 leader_uuid: "6cad901ab52a4c7b9a90110bcabd529d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } } }
03:08:03.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:03.688410 15593 sys_catalog.cc:458] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c [sys.catalog]: This master's current role is: FOLLOWER
03:08:03.706 [WARN - kudu-nio-0] (SecurityContext.java:206) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
03:08:03.727 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.14.36.61
03:08:03.736 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.14.36.62
03:08:03.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:03.856623 15505 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "6cad901ab52a4c7b9a90110bcabd529d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } } }
03:08:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:03.857708 15505 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [sys.catalog]: This master's current role is: LEADER
03:08:03.948 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.14.36.60
03:08:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:03.985272 14967 heartbeater.cc:344] Connected to a master server at 127.14.36.60:45755
03:08:03.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:03.989059 15552 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "1e74ae69837f4a19929b3bf70017a23d" instance_seqno: 1769656070527044) as {username='kudu', principal='kudu/127.14.36.1@KRBTEST.COM'} at 127.14.36.1:35887; Asking this server to re-register.
03:08:03.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:03.992584 14967 heartbeater.cc:461] Registering TS with master...
03:08:03.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:03.997180 15552 ts_manager.cc:194] Registered new tserver with Master: 1e74ae69837f4a19929b3bf70017a23d (127.14.36.1:45385)
03:08:04.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:04.043493 15117 heartbeater.cc:344] Connected to a master server at 127.14.36.60:45755
03:08:04.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:04.046391 15552 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "2bb5a457deff4c0ba8fcedcccfb2b50b" instance_seqno: 1769656072795400) as {username='kudu', principal='kudu/127.14.36.2@KRBTEST.COM'} at 127.14.36.2:39885; Asking this server to re-register.
03:08:04.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:04.048851 15117 heartbeater.cc:461] Registering TS with master...
03:08:04.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:04.051972 15552 ts_manager.cc:194] Registered new tserver with Master: 2bb5a457deff4c0ba8fcedcccfb2b50b (127.14.36.2:37651)
03:08:04.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:08:04 dist-test-slave-7fb5 krb5kdc[14493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656075, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.14.36.61@KRBTEST.COM
03:08:04.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:08:04 dist-test-slave-7fb5 krb5kdc[14493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656075, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.14.36.62@KRBTEST.COM
03:08:04.198 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xe81af46a, L:/127.0.0.1:56104 - R:/127.14.36.61:45213] using SASL/GSSAPI
03:08:04.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:08:04 dist-test-slave-7fb5 krb5kdc[14493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656075, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.14.36.60@KRBTEST.COM
03:08:04.245 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xca68b0ea, L:/127.0.0.1:49694 - R:/127.14.36.62:35391] using SASL/GSSAPI
03:08:04.377 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
03:08:04.388 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=1253), Traces: [0ms] refreshing cache from master, [50ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.36.62:35391, [233ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.36.61:45213, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.36.60:45755, [1083ms] Sub RPC ConnectToMaster: received response from server master-127.14.36.61:45213: OK, [1164ms] Sub RPC ConnectToMaster: received response from server master-127.14.36.62:35391: OK, deferred=null) after lookup
03:08:04.434 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xf0e3e56f, L:/127.0.0.1:43492 - R:/127.14.36.60:45755] using SASL/GSSAPI
03:08:04.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:04.880467 14323 debug-util.cc:398] Leaking SignalData structure 0x7b08000adde0 after lost signal to thread 14196
03:08:04.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:04.881484 14323 debug-util.cc:398] Leaking SignalData structure 0x7b08000adfe0 after lost signal to thread 14326
03:08:05.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:08:05 dist-test-slave-7fb5 krb5kdc[14493](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656085, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.3@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:05.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:05.026722 15147 init.cc:295] Successfully reacquired a new kerberos TGT
03:08:05.429 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
03:08:05.429 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
03:08:05.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:08:05 dist-test-slave-7fb5 krb5kdc[14493](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656085, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:05.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
03:08:05.472 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:08:05.473 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/krb5cc
03:08:05.474 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:08:05.475 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.14.36.62:35391
03:08:05.475 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
03:08:05.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:05.475631 14480 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu with pid 15278
03:08:05.520 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:08:05.520 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.14.36.61:45213
03:08:05.521 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
03:08:05.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:05.521703 14480 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu with pid 15364
03:08:05.525 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.14.36.62:35391(127.14.36.62:35391)] handling channelInactive
03:08:05.526 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.14.36.62:35391(127.14.36.62:35391)] cleaning up while in state READY due to: connection closed
03:08:05.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:05.584295 15155 connection.cc:537] client connection to 127.14.36.61:45213 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
03:08:05.585 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.14.36.61:45213(127.14.36.61:45213)] handling channelInactive
03:08:05.586 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.14.36.61:45213(127.14.36.61:45213)] cleaning up while in state READY due to: connection closed
03:08:05.592 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:08:05.592 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.14.36.60:45755
03:08:05.593 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
03:08:05.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:05.593461 14480 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu with pid 15469
03:08:05.616 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.14.36.60:45755(127.14.36.60:45755)] handling channelInactive
03:08:05.616 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.14.36.60:45755(127.14.36.60:45755)] cleaning up while in state READY due to: connection closed
03:08:05.617 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:08:05.617 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.14.36.62:35391
03:08:05.618 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
03:08:05.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:05.619534 14480 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu
03:08:05.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu
03:08:05.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/wal
03:08:05.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/data
03:08:05.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:08:05.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:08:05.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:08:05.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:05.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:05.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:05.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/logs
03:08:05.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/data/info.pb
03:08:05.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:05.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:08:05.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:08:05.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:08:05.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:08:05.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:05.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:05.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:05.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:08:05.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:08:05.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:05.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:05.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.36.62:35391
03:08:05.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.36.62
03:08:05.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37233
03:08:05.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab
03:08:05.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.36.62
03:08:05.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:08:05.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:08:05.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:08:05.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:08:05.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.36.20:43983
03:08:05.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:05.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:05.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:05.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:05.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.36.62:35391,127.14.36.61:45213,127.14.36.60:45755
03:08:05.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:05.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37645/jwks.json
03:08:05.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:08:05.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:08:05.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:05.954056 15631 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:08:05.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:05.954604 15631 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:08:05.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:05.955206 15631 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:08:05.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:05.988509 15631 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:08:05.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:05.988747 15631 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:08:05.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:05.989010 15631 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:08:05.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:05.989207 15631 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:08:05.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:05.989500 15631 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:08:05.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:05.989852 15631 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:08:05.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:05.990111 15631 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37645/jwks.json
03:08:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:06.033738 15631 master_runner.cc:386] Master server non-default flags:
03:08:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.36.20:43983
03:08:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/data
03:08:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/wal
03:08:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:08:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.36.62:35391,127.14.36.61:45213,127.14.36.60:45755
03:08:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab
03:08:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.36.62
03:08:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.36.62:35391
03:08:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:08:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:06.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37645/jwks.json
03:08:06.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:08:06.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:08:06.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:08:06.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:06.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:06.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/data/info.pb
03:08:06.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.36.62
03:08:06.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37233
03:08:06.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:08:06.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:06.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:08:06.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:08:06.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:06.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/logs
03:08:06.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:06.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:08:06.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:08:06.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:08:06.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:08:06.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
03:08:06.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:08:06.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Jan 2026 02:43:21 UTC on 5fd53c4cbb9d
03:08:06.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10241
03:08:06.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:08:06.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:06.035372 15631 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:08:06.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:06.036799 15631 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:08:06.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:06.049618 15641 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:07.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:07.450999 15640 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 15631
03:08:07.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:07.547477 15640 kernel_stack_watchdog.cc:198] Thread 15631 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 397ms:
03:08:07.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
03:08:07.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
03:08:07.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
03:08:07.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
03:08:07.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:06.050983 15642 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:07.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:07.554296 15631 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.502s user 0.002s sys 0.000s
03:08:07.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:07.554795 15631 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.503s user 0.002s sys 0.000s
03:08:07.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:07.561887 15644 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:07.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:07.562994 15631 server_base.cc:1060] running on GCE node
03:08:07.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:07.564265 15631 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:08:07.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:07.567392 15631 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:08:07.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:07.568861 15631 hybrid_clock.cc:648] HybridClock initialized: now 1769656087568796 us; error 59 us; skew 500 ppm
03:08:07.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:07.569813 15631 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:08:07.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:08:07 dist-test-slave-7fb5 krb5kdc[14493](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656087, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:07.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:07.576318 15631 init.cc:377] Logged in from keytab as kudu/127.14.36.62@KRBTEST.COM (short username kudu)
03:08:07.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:07.582880 15631 webserver.cc:492] Webserver started at http://127.14.36.62:37233/ using document root <none> and password file <none>
03:08:07.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:07.584043 15631 fs_manager.cc:362] Metadata directory not provided
03:08:07.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:07.584293 15631 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:08:07.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:07.593035 15631 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.008s sys 0.000s
03:08:07.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:07.597757 15657 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:08:07.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:07.599023 15631 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
03:08:07.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:07.599520 15631 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/data,/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/wal
03:08:07.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6cad901ab52a4c7b9a90110bcabd529d"
03:08:07.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-29 03:07:41 on dist-test-slave-7fb5"
03:08:07.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:07.601773 15631 fs_report.cc:389] FS layout report
03:08:07.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:08:07.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/wal
03:08:07.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/wal
03:08:07.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/data/data
03:08:07.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:08:07.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:08:07.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:08:07.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:08:07.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:08:07.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:08:07.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:08:07.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:08:07.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:08:07.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:08:07.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:08:07.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:07.662195 15631 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:08:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:07.668501 15631 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:08:07.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:07.669034 15631 kserver.cc:163] Server-wide thread pool size limit: 3276
03:08:07.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:07.744585 15631 rpc_server.cc:307] RPC server started. Bound to: 127.14.36.62:35391
03:08:07.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:07.744691 15710 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.14.36.62:35391 every 8 connection(s)
03:08:07.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:07.747639 15631 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/data/info.pb
03:08:07.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:07.749567 14480 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu as pid 15631
03:08:07.750 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:08:07.750 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.14.36.61:45213
03:08:07.751 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
03:08:07.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:07.752674 14480 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu
03:08:07.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu
03:08:07.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/wal
03:08:07.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/data
03:08:07.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:08:07.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:08:07.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:08:07.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:07.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:07.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:07.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/logs
03:08:07.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/data/info.pb
03:08:07.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:07.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:08:07.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:08:07.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:08:07.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:08:07.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:07.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:07.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:07.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:08:07.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:08:07.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:07.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:07.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.36.61:45213
03:08:07.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.36.61
03:08:07.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44571
03:08:07.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab
03:08:07.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.36.61
03:08:07.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:08:07.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:08:07.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:08:07.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:08:07.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.36.20:43983
03:08:07.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:07.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:07.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:07.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:07.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.36.62:35391,127.14.36.61:45213,127.14.36.60:45755
03:08:07.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:07.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37645/jwks.json
03:08:07.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:08:07.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:08:07.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:07.760957 15711 sys_catalog.cc:263] Verifying existing consensus state
03:08:07.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:07.766185 15711 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d: Bootstrap starting.
03:08:07.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:07.807826 15711 log.cc:826] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d: Log is configured to *not* fsync() on all Append() calls
03:08:07.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:07.824810 15711 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:08:07.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:07.825721 15711 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d: Bootstrap complete.
03:08:07.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:07.861696 15711 raft_consensus.cc:359] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } }
03:08:07.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:07.864710 15711 raft_consensus.cc:740] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6cad901ab52a4c7b9a90110bcabd529d, State: Initialized, Role: FOLLOWER
03:08:07.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:07.865842 15711 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } }
03:08:07.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:08:07 dist-test-slave-7fb5 krb5kdc[14493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656080, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.1@KRBTEST.COM for kudu/127.14.36.62@KRBTEST.COM
03:08:07.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:07.886754 15717 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } } }
03:08:07.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:07.889832 15717 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [sys.catalog]: This master's current role is: FOLLOWER
03:08:07.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:07.888718 15711 sys_catalog.cc:565] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [sys.catalog]: configured and running, proceeding with master startup.
03:08:07.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:07.892844 14969 heartbeater.cc:344] Connected to a master server at 127.14.36.62:35391
03:08:07.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:07.923453 15728 catalog_manager.cc:1269] Loaded cluster ID: 5d811d19a7434727a35b22e3a8e4d9bd
03:08:07.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:07.926329 15728 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d: loading cluster ID for follower catalog manager: success
03:08:07.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:07.934340 15728 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d: acquiring CA information for follower catalog manager: success
03:08:07.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:08:07 dist-test-slave-7fb5 krb5kdc[14493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656087, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.62@KRBTEST.COM for kudu/127.14.36.62@KRBTEST.COM
03:08:07.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:07.941694 15728 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:08:07.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:07.962957 15631 master_runner.cc:186] Error getting master registration for 127.14.36.61:45213: Network error: Client connection negotiation failed: client connection to 127.14.36.61:45213: connect: Connection refused (error 111),
03:08:07.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:07.992506 15631 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:08:08.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:08.057487 15119 heartbeater.cc:344] Connected to a master server at 127.14.36.62:35391
03:08:08.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:08.064097 15675 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "2bb5a457deff4c0ba8fcedcccfb2b50b" instance_seqno: 1769656072795400) as {username='kudu', principal='kudu/127.14.36.2@KRBTEST.COM'} at 127.14.36.2:51399; Asking this server to re-register.
03:08:08.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:08.065960 15119 heartbeater.cc:461] Registering TS with master...
03:08:08.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:08.070135 15675 ts_manager.cc:194] Registered new tserver with Master: 2bb5a457deff4c0ba8fcedcccfb2b50b (127.14.36.2:37651)
03:08:08.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:08.149091 15713 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:08:08.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:08.149780 15713 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:08:08.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:08.150556 15713 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:08:08.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:08.184980 15713 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:08:08.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:08.185271 15713 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:08:08.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:08.185655 15713 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:08:08.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:08.185988 15713 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:08:08.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:08.186300 15713 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:08:08.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:08.186625 15713 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:08:08.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:08.186882 15713 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37645/jwks.json
03:08:08.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:08.246279 15713 master_runner.cc:386] Master server non-default flags:
03:08:08.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:08.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.36.20:43983
03:08:08.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:08.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:08.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/data
03:08:08.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/wal
03:08:08.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:08:08.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:08.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.36.62:35391,127.14.36.61:45213,127.14.36.60:45755
03:08:08.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab
03:08:08.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.36.61
03:08:08.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:08.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:08.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:08.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.36.61:45213
03:08:08.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:08.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:08:08.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:08.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37645/jwks.json
03:08:08.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:08:08.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:08:08.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:08:08.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:08.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:08.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/data/info.pb
03:08:08.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.36.61
03:08:08.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44571
03:08:08.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:08:08.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:08.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:08:08.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:08:08.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:08.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/logs
03:08:08.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:08.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:08:08.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:08:08.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:08:08.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:08:08.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
03:08:08.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:08:08.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Jan 2026 02:43:21 UTC on 5fd53c4cbb9d
03:08:08.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10241
03:08:08.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:08:08.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:08.248008 15713 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:08:08.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:08.249388 15713 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:08:08.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:08.262310 15737 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:08.528630 15266 heartbeater.cc:344] Connected to a master server at 127.14.36.62:35391
03:08:08.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:08.531620 15675 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "a0016e6f9c82495288bb9911310fe995" instance_seqno: 1769656075243095) as {username='kudu', principal='kudu/127.14.36.3@KRBTEST.COM'} at 127.14.36.3:44563; Asking this server to re-register.
03:08:08.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:08.533624 15266 heartbeater.cc:461] Registering TS with master...
03:08:08.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:08.537307 15675 ts_manager.cc:194] Registered new tserver with Master: a0016e6f9c82495288bb9911310fe995 (127.14.36.3:34141)
03:08:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:08.898984 15675 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "1e74ae69837f4a19929b3bf70017a23d" instance_seqno: 1769656070527044) as {username='kudu', principal='kudu/127.14.36.1@KRBTEST.COM'} at 127.14.36.1:53879; Asking this server to re-register.
03:08:08.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:08.900879 14969 heartbeater.cc:461] Registering TS with master...
03:08:08.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:08.904114 15675 ts_manager.cc:194] Registered new tserver with Master: 1e74ae69837f4a19929b3bf70017a23d (127.14.36.1:45385)
03:08:09.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:09.009438 15744 raft_consensus.cc:493] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:08:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:09.010644 15744 raft_consensus.cc:515] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } }
03:08:09.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:09.020830 15744 leader_election.cc:290] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers de1dd59a869a46bdb0165c8963557689 (127.14.36.61:45213), a5feff59896e4438ac3b32435885f30c (127.14.36.60:45755)
03:08:09.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:09.035315 15662 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.14.36.61:45213: connect: Connection refused (error 111)
03:08:09.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:09.054061 15662 leader_election.cc:336] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer de1dd59a869a46bdb0165c8963557689 (127.14.36.61:45213): Network error: Client connection negotiation failed: client connection to 127.14.36.61:45213: connect: Connection refused (error 111)
03:08:09.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:09.055121 15662 leader_election.cc:336] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer a5feff59896e4438ac3b32435885f30c (127.14.36.60:45755): Network error: Client connection negotiation failed: client connection to 127.14.36.60:45755: connect: Connection refused (error 111)
03:08:09.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:09.055698 15662 leader_election.cc:304] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6cad901ab52a4c7b9a90110bcabd529d; no voters: a5feff59896e4438ac3b32435885f30c, de1dd59a869a46bdb0165c8963557689
03:08:09.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:09.056857 15744 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
03:08:09.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:09.663928 15736 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 15713
03:08:09.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:09.878142 15736 kernel_stack_watchdog.cc:198] Thread 15713 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
03:08:09.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
03:08:09.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
03:08:09.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
03:08:09.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
03:08:09.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:08.263449 15738 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:09.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:09.878850 15713 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.617s user 0.581s sys 0.967s
03:08:09.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:09.879495 15713 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.618s user 0.582s sys 0.967s
03:08:09.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:09.892458 15751 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:09.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:09.893455 15713 server_base.cc:1060] running on GCE node
03:08:09.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:09.894637 15713 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:08:09.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:09.897786 15713 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:08:09.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:09.899259 15713 hybrid_clock.cc:648] HybridClock initialized: now 1769656089899136 us; error 115 us; skew 500 ppm
03:08:09.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:09.900316 15713 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:08:09.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:08:09 dist-test-slave-7fb5 krb5kdc[14493](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656089, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:09.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:09.907578 15713 init.cc:377] Logged in from keytab as kudu/127.14.36.61@KRBTEST.COM (short username kudu)
03:08:09.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:09.913988 15713 webserver.cc:492] Webserver started at http://127.14.36.61:44571/ using document root <none> and password file <none>
03:08:09.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:09.915068 15713 fs_manager.cc:362] Metadata directory not provided
03:08:09.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:09.915279 15713 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:08:09.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:09.924142 15713 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.006s sys 0.001s
03:08:09.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:09.929315 15757 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:08:09.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:09.930696 15713 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
03:08:09.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:09.931092 15713 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/data,/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/wal
03:08:09.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "de1dd59a869a46bdb0165c8963557689"
03:08:09.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-29 03:07:43 on dist-test-slave-7fb5"
03:08:09.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:09.933125 15713 fs_report.cc:389] FS layout report
03:08:09.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:08:09.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/wal
03:08:09.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/wal
03:08:09.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/data/data
03:08:09.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:08:09.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:08:09.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:08:09.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:08:09.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:08:09.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:08:09.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:08:09.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:08:09.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:08:09.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:08:09.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:08:10.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:10.000648 15713 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:08:10.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:10.011531 15713 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:08:10.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:10.012528 15713 kserver.cc:163] Server-wide thread pool size limit: 3276
03:08:10.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:10.116339 15713 rpc_server.cc:307] RPC server started. Bound to: 127.14.36.61:45213
03:08:10.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:10.116371 15812 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.14.36.61:45213 every 8 connection(s)
03:08:10.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:10.120539 15713 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/data/info.pb
03:08:10.126 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:08:10.126 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.14.36.60:45755
03:08:10.126 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
03:08:10.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:10.125228 14480 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu as pid 15713
03:08:10.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:10.131327 14480 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu
03:08:10.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu
03:08:10.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/wal
03:08:10.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/data
03:08:10.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:08:10.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:08:10.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:08:10.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:10.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:10.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:10.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/logs
03:08:10.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/data/info.pb
03:08:10.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:10.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:08:10.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:08:10.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:08:10.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:08:10.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:10.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:10.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:10.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:08:10.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:08:10.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:10.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:10.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.36.60:45755
03:08:10.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.36.60
03:08:10.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41699
03:08:10.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab
03:08:10.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.36.60
03:08:10.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:08:10.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:08:10.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:08:10.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:08:10.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.36.20:43983
03:08:10.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:10.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:10.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:10.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:10.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.36.62:35391,127.14.36.61:45213,127.14.36.60:45755
03:08:10.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:10.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37645/jwks.json
03:08:10.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:08:10.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:08:10.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:08:10 dist-test-slave-7fb5 krb5kdc[14493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656082, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.2@KRBTEST.COM for kudu/127.14.36.61@KRBTEST.COM
03:08:10.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:10.145800 15813 sys_catalog.cc:263] Verifying existing consensus state
03:08:10.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:10.155411 15813 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689: Bootstrap starting.
03:08:10.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:10.183971 15118 heartbeater.cc:344] Connected to a master server at 127.14.36.61:45213
03:08:10.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:10.217410 15813 log.cc:826] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689: Log is configured to *not* fsync() on all Append() calls
03:08:10.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:08:10 dist-test-slave-7fb5 krb5kdc[14493](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656090, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.1@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:10.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:10.245663 14848 init.cc:295] Successfully reacquired a new kerberos TGT
03:08:10.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:10.262884 15813 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:08:10.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:10.264014 15813 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689: Bootstrap complete.
03:08:10.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:10.294217 15813 raft_consensus.cc:359] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } }
03:08:10.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:10.297623 15813 raft_consensus.cc:740] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: de1dd59a869a46bdb0165c8963557689, State: Initialized, Role: FOLLOWER
03:08:10.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:10.298606 15813 consensus_queue.cc:260] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } }
03:08:10.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:10.303469 15818 sys_catalog.cc:455] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } } }
03:08:10.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:10.304661 15818 sys_catalog.cc:458] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [sys.catalog]: This master's current role is: FOLLOWER
03:08:10.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:10.305416 15813 sys_catalog.cc:565] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [sys.catalog]: configured and running, proceeding with master startup.
03:08:10.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:10.394621 15829 catalog_manager.cc:1269] Loaded cluster ID: 5d811d19a7434727a35b22e3a8e4d9bd
03:08:10.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:10.394909 15829 catalog_manager.cc:1562] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689: loading cluster ID for follower catalog manager: success
03:08:10.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:08:10 dist-test-slave-7fb5 krb5kdc[14493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656089, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.61@KRBTEST.COM for kudu/127.14.36.62@KRBTEST.COM
03:08:10.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:10.419566 15829 catalog_manager.cc:1584] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689: acquiring CA information for follower catalog manager: success
03:08:10.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:10.439549 15829 catalog_manager.cc:1612] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:08:10.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:10.468416 15713 master_runner.cc:186] Error getting master registration for 127.14.36.60:45755: Network error: Client connection negotiation failed: client connection to 127.14.36.60:45755: connect: Connection refused (error 111),
03:08:10.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:10.498240 15713 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:08:10.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:10.627949 15265 heartbeater.cc:344] Connected to a master server at 127.14.36.61:45213
03:08:10.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:10.634194 15776 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "a0016e6f9c82495288bb9911310fe995" instance_seqno: 1769656075243095) as {username='kudu', principal='kudu/127.14.36.3@KRBTEST.COM'} at 127.14.36.3:39339; Asking this server to re-register.
03:08:10.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:10.636160 15265 heartbeater.cc:461] Registering TS with master...
03:08:10.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:10.639976 15775 ts_manager.cc:194] Registered new tserver with Master: a0016e6f9c82495288bb9911310fe995 (127.14.36.3:34141)
03:08:10.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:10.654246 15816 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:08:10.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:10.654930 15816 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:08:10.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:10.655681 15816 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:08:10.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:10.694011 15816 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:08:10.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:10.694293 15816 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:08:10.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:10.694734 15816 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:08:10.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:10.695039 15816 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:08:10.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:10.695367 15816 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:08:10.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:10.695677 15816 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:08:10.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:10.695927 15816 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37645/jwks.json
03:08:10.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:10.739632 15816 master_runner.cc:386] Master server non-default flags:
03:08:10.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:10.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.36.20:43983
03:08:10.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:10.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:10.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/data
03:08:10.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/wal
03:08:10.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:08:10.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:10.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.36.62:35391,127.14.36.61:45213,127.14.36.60:45755
03:08:10.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab
03:08:10.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.36.60
03:08:10.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:10.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:10.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:10.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.36.60:45755
03:08:10.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:10.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:08:10.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:10.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37645/jwks.json
03:08:10.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:08:10.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:08:10.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:08:10.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:10.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:10.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/data/info.pb
03:08:10.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.36.60
03:08:10.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41699
03:08:10.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:08:10.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:10.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:08:10.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:08:10.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:10.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/logs
03:08:10.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:10.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:08:10.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:08:10.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:08:10.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:08:10.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
03:08:10.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:08:10.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Jan 2026 02:43:21 UTC on 5fd53c4cbb9d
03:08:10.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10241
03:08:10.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:08:10.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:10.741380 15816 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:08:10.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:10.742668 15816 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:08:10.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:10.781675 15840 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:10.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:10.873481 15846 raft_consensus.cc:493] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:08:10.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:10.874480 15846 raft_consensus.cc:515] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } }
03:08:10.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:10.895628 15846 leader_election.cc:290] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers de1dd59a869a46bdb0165c8963557689 (127.14.36.61:45213), a5feff59896e4438ac3b32435885f30c (127.14.36.60:45755)
03:08:10.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:10.954958 15787 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6cad901ab52a4c7b9a90110bcabd529d" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "de1dd59a869a46bdb0165c8963557689" is_pre_election: true
03:08:10.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:10.956239 15787 raft_consensus.cc:2468] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6cad901ab52a4c7b9a90110bcabd529d in term 2.
03:08:10.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:10.959965 15662 leader_election.cc:304] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6cad901ab52a4c7b9a90110bcabd529d, de1dd59a869a46bdb0165c8963557689; no voters:
03:08:10.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:10.961251 15846 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [term 2 FOLLOWER]: Leader pre-election won for term 3
03:08:10.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:10.961695 15846 raft_consensus.cc:493] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:08:10.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:10.962059 15846 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [term 2 FOLLOWER]: Advancing to term 3
03:08:10.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:10.980345 15662 leader_election.cc:336] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer a5feff59896e4438ac3b32435885f30c (127.14.36.60:45755): Network error: Client connection negotiation failed: client connection to 127.14.36.60:45755: connect: Connection refused (error 111)
03:08:10.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:10.991065 15846 raft_consensus.cc:515] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } }
03:08:10.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:10.995555 15787 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6cad901ab52a4c7b9a90110bcabd529d" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "de1dd59a869a46bdb0165c8963557689"
03:08:10.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:10.996397 15787 raft_consensus.cc:3060] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [term 2 FOLLOWER]: Advancing to term 3
03:08:11.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:11.018183 15662 leader_election.cc:336] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer a5feff59896e4438ac3b32435885f30c (127.14.36.60:45755): Network error: Client connection negotiation failed: client connection to 127.14.36.60:45755: connect: Connection refused (error 111)
03:08:11.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:11.019459 15846 leader_election.cc:290] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [CANDIDATE]: Term 3 election: Requested vote from peers de1dd59a869a46bdb0165c8963557689 (127.14.36.61:45213), a5feff59896e4438ac3b32435885f30c (127.14.36.60:45755)
03:08:11.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:11.030349 15787 raft_consensus.cc:2468] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6cad901ab52a4c7b9a90110bcabd529d in term 3.
03:08:11.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:11.034502 15662 leader_election.cc:304] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6cad901ab52a4c7b9a90110bcabd529d, de1dd59a869a46bdb0165c8963557689; no voters: a5feff59896e4438ac3b32435885f30c
03:08:11.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:11.038069 15846 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [term 3 FOLLOWER]: Leader election won for term 3
03:08:11.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:11.059105 15846 raft_consensus.cc:697] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [term 3 LEADER]: Becoming Leader. State: Replica: 6cad901ab52a4c7b9a90110bcabd529d, State: Running, Role: LEADER
03:08:11.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:11.060905 14968 heartbeater.cc:344] Connected to a master server at 127.14.36.61:45213
03:08:11.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:11.062352 15846 consensus_queue.cc:237] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } }
03:08:11.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:11.064847 15775 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "1e74ae69837f4a19929b3bf70017a23d" instance_seqno: 1769656070527044) as {username='kudu', principal='kudu/127.14.36.1@KRBTEST.COM'} at 127.14.36.1:60479; Asking this server to re-register.
03:08:11.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:11.067317 14968 heartbeater.cc:461] Registering TS with master...
03:08:11.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:11.071743 15775 ts_manager.cc:194] Registered new tserver with Master: 1e74ae69837f4a19929b3bf70017a23d (127.14.36.1:45385)
03:08:11.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:11.155339 15852 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6cad901ab52a4c7b9a90110bcabd529d. Latest consensus state: current_term: 3 leader_uuid: "6cad901ab52a4c7b9a90110bcabd529d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } } }
03:08:11.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:11.160533 15852 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [sys.catalog]: This master's current role is: LEADER
03:08:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:11.191426 15775 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "2bb5a457deff4c0ba8fcedcccfb2b50b" instance_seqno: 1769656072795400) as {username='kudu', principal='kudu/127.14.36.2@KRBTEST.COM'} at 127.14.36.2:36453; Asking this server to re-register.
03:08:11.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:11.193635 15118 heartbeater.cc:461] Registering TS with master...
03:08:11.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:11.194895 15856 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
03:08:11.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:11.197279 15775 ts_manager.cc:194] Registered new tserver with Master: 2bb5a457deff4c0ba8fcedcccfb2b50b (127.14.36.2:37651)
03:08:11.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:11.278156 15856 catalog_manager.cc:1494] Initializing Kudu cluster ID...
03:08:11.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:11.335350 15856 catalog_manager.cc:1269] Loaded cluster ID: 5d811d19a7434727a35b22e3a8e4d9bd
03:08:11.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:11.335712 15856 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
03:08:11.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:11.357175 15856 catalog_manager.cc:1514] Loading token signing keys...
03:08:11.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:11.402951 15856 catalog_manager.cc:6038] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d: Loaded TSK: 0
03:08:11.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:11.408646 15856 catalog_manager.cc:1524] Initializing in-progress tserver states...
03:08:11.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:11.475126 15787 raft_consensus.cc:1275] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [term 3 FOLLOWER]: Refusing update from remote peer 6cad901ab52a4c7b9a90110bcabd529d: Log matching property violated. Preceding OpId in replica: term: 2 index: 5. Preceding OpId from leader: term: 3 index: 6. (index mismatch)
03:08:11.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:11.481604 15662 consensus_peers.cc:597] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d -> Peer a5feff59896e4438ac3b32435885f30c (127.14.36.60:45755): Couldn't send request to peer a5feff59896e4438ac3b32435885f30c. Status: Network error: Client connection negotiation failed: client connection to 127.14.36.60:45755: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
03:08:11.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:11.482570 15852 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [LEADER]: Connected to new peer: Peer: permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
03:08:11.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:11.535853 15857 sys_catalog.cc:455] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6cad901ab52a4c7b9a90110bcabd529d. Latest consensus state: current_term: 3 leader_uuid: "6cad901ab52a4c7b9a90110bcabd529d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } } }
03:08:11.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:11.536805 15857 sys_catalog.cc:458] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [sys.catalog]: This master's current role is: FOLLOWER
03:08:11.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:11.553828 15846 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "6cad901ab52a4c7b9a90110bcabd529d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } } }
03:08:11.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:11.554594 15846 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [sys.catalog]: This master's current role is: LEADER
03:08:11.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:11.566705 15857 sys_catalog.cc:455] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "6cad901ab52a4c7b9a90110bcabd529d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } } }
03:08:11.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:11.567760 15857 sys_catalog.cc:458] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [sys.catalog]: This master's current role is: FOLLOWER
03:08:12.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:12.557873 15266 heartbeater.cc:499] Master 127.14.36.62:35391 was elected leader, sending a full tablet report...
03:08:12.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:08:12 dist-test-slave-7fb5 krb5kdc[14493](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656092, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.2@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:12.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:12.592768 14998 init.cc:295] Successfully reacquired a new kerberos TGT
03:08:12.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:12.918792 14969 heartbeater.cc:499] Master 127.14.36.62:35391 was elected leader, sending a full tablet report...
03:08:13.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:13.009905 15631 master_runner.cc:186] Error getting master registration for 127.14.36.60:45755: Network error: Client connection negotiation failed: client connection to 127.14.36.60:45755: connect: Connection refused (error 111),
03:08:13.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:13.010434 15631 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:08:13.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:13.179281 15119 heartbeater.cc:499] Master 127.14.36.62:35391 was elected leader, sending a full tablet report...
03:08:13.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:10.788337 15841 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:13.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:12.166905 15839 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 15816
03:08:13.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:13.940495 15816 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 3.152s user 1.034s sys 1.708s
03:08:13.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:13.945360 15816 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 3.157s user 1.035s sys 1.708s
03:08:13.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:13.947249 15843 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:13.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:13.949249 15662 consensus_peers.cc:597] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d -> Peer a5feff59896e4438ac3b32435885f30c (127.14.36.60:45755): Couldn't send request to peer a5feff59896e4438ac3b32435885f30c. Status: Network error: Client connection negotiation failed: client connection to 127.14.36.60:45755: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
03:08:13.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:13.949537 15816 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:08:13.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:13.949445 15842 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 3156 milliseconds
03:08:13.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:13.950893 15816 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:08:13.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:13.954303 15816 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:08:13.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:13.955669 15816 hybrid_clock.cc:648] HybridClock initialized: now 1769656093955627 us; error 57 us; skew 500 ppm
03:08:13.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:13.956557 15816 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:08:13.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:08:13 dist-test-slave-7fb5 krb5kdc[14493](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656093, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:13.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:13.962646 15816 init.cc:377] Logged in from keytab as kudu/127.14.36.60@KRBTEST.COM (short username kudu)
03:08:13.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:13.967590 15816 webserver.cc:492] Webserver started at http://127.14.36.60:41699/ using document root <none> and password file <none>
03:08:13.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:13.968511 15816 fs_manager.cc:362] Metadata directory not provided
03:08:13.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:13.968755 15816 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:08:13.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:13.976992 15816 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.006s sys 0.001s
03:08:13.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:13.982065 15883 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:08:13.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:13.983418 15816 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
03:08:13.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:13.983791 15816 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/data,/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/wal
03:08:13.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a5feff59896e4438ac3b32435885f30c"
03:08:13.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-29 03:07:45 on dist-test-slave-7fb5"
03:08:13.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:13.985780 15816 fs_report.cc:389] FS layout report
03:08:13.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:08:13.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/wal
03:08:13.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/wal
03:08:13.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/data/data
03:08:13.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:08:13.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:08:13.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:08:13.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:08:13.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:08:13.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:08:13.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:08:13.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:08:13.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:08:13.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:08:13.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:08:14.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:14.047394 15816 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:08:14.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:14.053895 15816 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:08:14.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:14.054414 15816 kserver.cc:163] Server-wide thread pool size limit: 3276
03:08:14.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:14.153677 15816 rpc_server.cc:307] RPC server started. Bound to: 127.14.36.60:45755
03:08:14.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:14.153782 15937 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.14.36.60:45755 every 8 connection(s)
03:08:14.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:14.157794 15816 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/data/info.pb
03:08:14.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:14.167407 14480 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu as pid 15816
03:08:14.168 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:08:14.169 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.14.36.62:35391(127.14.36.62:35391)] connecting to peer
03:08:14.172 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.14.36.62:35391(127.14.36.62:35391)] Successfully connected to peer
03:08:14.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:14.174558 15938 sys_catalog.cc:263] Verifying existing consensus state
03:08:14.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:14.182755 15938 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c: Bootstrap starting.
03:08:14.188 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.14.36.62
03:08:14.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:08:14 dist-test-slave-7fb5 krb5kdc[14493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656092, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.2@KRBTEST.COM for kudu/127.14.36.60@KRBTEST.COM
03:08:14.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:08:14 dist-test-slave-7fb5 krb5kdc[14493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656085, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.14.36.62@KRBTEST.COM
03:08:14.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:14.271993 15938 log.cc:826] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c: Log is configured to *not* fsync() on all Append() calls
03:08:14.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:14.291311 15117 heartbeater.cc:344] Connected to a master server at 127.14.36.60:45755
03:08:14.292 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x5f34fe9f, L:/127.0.0.1:50526 - R:/127.14.36.62:35391] using SASL/GSSAPI
03:08:14.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:14.299722 15938 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:08:14.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:14.300632 15938 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c: Bootstrap complete.
03:08:14.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:14.322788 15938 raft_consensus.cc:359] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } }
03:08:14.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:14.326306 15938 raft_consensus.cc:740] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: a5feff59896e4438ac3b32435885f30c, State: Initialized, Role: FOLLOWER
03:08:14.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:14.327368 15938 consensus_queue.cc:260] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } }
03:08:14.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:14.330382 15944 sys_catalog.cc:455] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } } }
03:08:14.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:14.331364 15944 sys_catalog.cc:458] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c [sys.catalog]: This master's current role is: FOLLOWER
03:08:14.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:14.332332 15938 sys_catalog.cc:565] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c [sys.catalog]: configured and running, proceeding with master startup.
03:08:14.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:14.363171 15956 catalog_manager.cc:1269] Loaded cluster ID: 5d811d19a7434727a35b22e3a8e4d9bd
03:08:14.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:14.363468 15956 catalog_manager.cc:1562] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c: loading cluster ID for follower catalog manager: success
03:08:14.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:14.391127 15956 catalog_manager.cc:1584] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c: acquiring CA information for follower catalog manager: success
03:08:14.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:08:14 dist-test-slave-7fb5 krb5kdc[14493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656093, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.60@KRBTEST.COM for kudu/127.14.36.62@KRBTEST.COM
03:08:14.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:14.396679 15956 catalog_manager.cc:1612] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:08:14.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:14.542485 15912 raft_consensus.cc:3060] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c [term 2 FOLLOWER]: Advancing to term 3
03:08:14.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:14.602133 15963 sys_catalog.cc:455] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "6cad901ab52a4c7b9a90110bcabd529d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } } }
03:08:14.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:14.604228 15963 sys_catalog.cc:458] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c [sys.catalog]: This master's current role is: FOLLOWER
03:08:14.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:14.606272 15944 sys_catalog.cc:455] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6cad901ab52a4c7b9a90110bcabd529d. Latest consensus state: current_term: 3 leader_uuid: "6cad901ab52a4c7b9a90110bcabd529d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } } }
03:08:14.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:14.606961 15944 sys_catalog.cc:458] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c [sys.catalog]: This master's current role is: FOLLOWER
03:08:14.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:14.787256 15264 heartbeater.cc:344] Connected to a master server at 127.14.36.60:45755
03:08:14.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:14.792531 15902 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "a0016e6f9c82495288bb9911310fe995" instance_seqno: 1769656075243095) as {username='kudu', principal='kudu/127.14.36.3@KRBTEST.COM'} at 127.14.36.3:57329; Asking this server to re-register.
03:08:14.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:14.794023 15264 heartbeater.cc:461] Registering TS with master...
03:08:14.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:14.797664 15902 ts_manager.cc:194] Registered new tserver with Master: a0016e6f9c82495288bb9911310fe995 (127.14.36.3:34141)
03:08:15.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:08:15 dist-test-slave-7fb5 krb5kdc[14493](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656095, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.3@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:15.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:15.029717 15147 init.cc:295] Successfully reacquired a new kerberos TGT
03:08:15.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:15.104281 14967 heartbeater.cc:344] Connected to a master server at 127.14.36.60:45755
03:08:15.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:15.106638 15902 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "1e74ae69837f4a19929b3bf70017a23d" instance_seqno: 1769656070527044) as {username='kudu', principal='kudu/127.14.36.1@KRBTEST.COM'} at 127.14.36.1:51451; Asking this server to re-register.
03:08:15.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:15.107846 14967 heartbeater.cc:461] Registering TS with master...
03:08:15.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:15.110217 15902 ts_manager.cc:194] Registered new tserver with Master: 1e74ae69837f4a19929b3bf70017a23d (127.14.36.1:45385)
03:08:15.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:15.140198 15966 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "6cad901ab52a4c7b9a90110bcabd529d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } } }
03:08:15.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:15.141980 15966 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [sys.catalog]: This master's current role is: LEADER
03:08:15.299 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
03:08:15.300 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
03:08:15.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:15.299913 15902 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "2bb5a457deff4c0ba8fcedcccfb2b50b" instance_seqno: 1769656072795400) as {username='kudu', principal='kudu/127.14.36.2@KRBTEST.COM'} at 127.14.36.2:34937; Asking this server to re-register.
03:08:15.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:15.302376 15117 heartbeater.cc:461] Registering TS with master...
03:08:15.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:15.306288 15902 ts_manager.cc:194] Registered new tserver with Master: 2bb5a457deff4c0ba8fcedcccfb2b50b (127.14.36.2:37651)
03:08:15.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:08:15 dist-test-slave-7fb5 krb5kdc[14493](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656095, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:15.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
03:08:15.333 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:08:15.333 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/krb5cc
03:08:15.334 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:08:15.334 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.14.36.62:35391
03:08:15.335 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
03:08:15.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:15.335486 14480 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu with pid 15631
03:08:15.367 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.14.36.62:35391(127.14.36.62:35391)] handling channelInactive
03:08:15.367 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.14.36.62:35391(127.14.36.62:35391)] cleaning up while in state READY due to: connection closed
03:08:15.370 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:08:15.370 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.14.36.61:45213
03:08:15.370 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
03:08:15.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:15.370992 14480 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu with pid 15713
03:08:15.404 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:08:15.404 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.14.36.60:45755
03:08:15.405 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
03:08:15.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:15.406154 14480 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu with pid 15816
03:08:15.427 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:08:15.428 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.14.36.62:35391
03:08:15.428 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
03:08:15.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:15.429894 14480 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu
03:08:15.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu
03:08:15.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/wal
03:08:15.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/data
03:08:15.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:08:15.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:08:15.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:08:15.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:15.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:15.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:15.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/logs
03:08:15.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/data/info.pb
03:08:15.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:15.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:08:15.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:08:15.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:08:15.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:08:15.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:15.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:15.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:15.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:08:15.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:08:15.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:15.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:15.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.36.62:35391
03:08:15.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.36.62
03:08:15.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37233
03:08:15.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab
03:08:15.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.36.62
03:08:15.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:08:15.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:08:15.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:08:15.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:08:15.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.36.20:43983
03:08:15.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:15.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:15.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:15.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:15.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.36.62:35391,127.14.36.61:45213,127.14.36.60:45755
03:08:15.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:15.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37645/jwks.json
03:08:15.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:08:15.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:08:15.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:15.761391 15972 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:08:15.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:15.761933 15972 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:08:15.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:15.762558 15972 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:08:15.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:15.795665 15972 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:08:15.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:15.795943 15972 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:08:15.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:15.796236 15972 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:08:15.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:15.796447 15972 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:08:15.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:15.796641 15972 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:08:15.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:15.796835 15972 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:08:15.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:15.796980 15972 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37645/jwks.json
03:08:15.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:15.843261 15972 master_runner.cc:386] Master server non-default flags:
03:08:15.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:15.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.36.20:43983
03:08:15.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:15.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:15.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/data
03:08:15.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/wal
03:08:15.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:08:15.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:15.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.36.62:35391,127.14.36.61:45213,127.14.36.60:45755
03:08:15.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab
03:08:15.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.36.62
03:08:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.36.62:35391
03:08:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:08:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37645/jwks.json
03:08:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:08:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:08:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:08:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/data/info.pb
03:08:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.36.62
03:08:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37233
03:08:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:08:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:08:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:08:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/logs
03:08:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:08:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:08:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:08:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:08:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
03:08:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:08:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Jan 2026 02:43:21 UTC on 5fd53c4cbb9d
03:08:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10241
03:08:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:08:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:15.845013 15972 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:08:15.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:15.846391 15972 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:08:15.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:15.857846 15981 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:17.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:15.859679 15982 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:17.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:17.232833 15984 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:17.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.235935 15972 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:08:17.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:17.236421 15983 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1376 milliseconds
03:08:17.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.237943 15972 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:08:17.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.241595 15972 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:08:17.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.243113 15972 hybrid_clock.cc:648] HybridClock initialized: now 1769656097243075 us; error 46 us; skew 500 ppm
03:08:17.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.244292 15972 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:08:17.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:08:17 dist-test-slave-7fb5 krb5kdc[14493](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656097, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:17.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.251600 15972 init.cc:377] Logged in from keytab as kudu/127.14.36.62@KRBTEST.COM (short username kudu)
03:08:17.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.256433 15972 webserver.cc:492] Webserver started at http://127.14.36.62:37233/ using document root <none> and password file <none>
03:08:17.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.257355 15972 fs_manager.cc:362] Metadata directory not provided
03:08:17.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.257555 15972 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:08:17.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.265566 15972 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.000s sys 0.004s
03:08:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.270254 15998 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:08:17.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.271507 15972 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
03:08:17.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.271828 15972 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/data,/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/wal
03:08:17.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6cad901ab52a4c7b9a90110bcabd529d"
03:08:17.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-29 03:07:41 on dist-test-slave-7fb5"
03:08:17.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.274354 15972 fs_report.cc:389] FS layout report
03:08:17.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:08:17.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/wal
03:08:17.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/wal
03:08:17.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/data/data
03:08:17.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:08:17.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:08:17.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:08:17.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:08:17.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:08:17.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:08:17.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:08:17.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:08:17.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:08:17.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:08:17.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:08:17.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.332468 15972 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:08:17.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.338419 15972 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:08:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.338843 15972 kserver.cc:163] Server-wide thread pool size limit: 3276
03:08:17.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.418887 15972 rpc_server.cc:307] RPC server started. Bound to: 127.14.36.62:35391
03:08:17.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.418980 16049 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.14.36.62:35391 every 8 connection(s)
03:08:17.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.421747 15972 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-0/data/info.pb
03:08:17.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.424741 14480 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu as pid 15972
03:08:17.425 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:08:17.425 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.14.36.61:45213
03:08:17.426 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
03:08:17.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.427392 14480 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu
03:08:17.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu
03:08:17.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/wal
03:08:17.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/data
03:08:17.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:08:17.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:08:17.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:08:17.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:17.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:17.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:17.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/logs
03:08:17.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/data/info.pb
03:08:17.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:17.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:08:17.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:08:17.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:08:17.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:08:17.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:17.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:17.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:17.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:08:17.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:08:17.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:17.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:17.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.36.61:45213
03:08:17.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.36.61
03:08:17.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44571
03:08:17.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab
03:08:17.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.36.61
03:08:17.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:08:17.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:08:17.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:08:17.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:08:17.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.36.20:43983
03:08:17.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:17.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:17.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:17.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:17.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.36.62:35391,127.14.36.61:45213,127.14.36.60:45755
03:08:17.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:17.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37645/jwks.json
03:08:17.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:08:17.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:08:17.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.435187 16050 sys_catalog.cc:263] Verifying existing consensus state
03:08:17.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.442469 16050 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d: Bootstrap starting.
03:08:17.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.484294 16050 log.cc:826] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d: Log is configured to *not* fsync() on all Append() calls
03:08:17.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.502621 16050 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:08:17.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.503542 16050 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d: Bootstrap complete.
03:08:17.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.531570 16050 raft_consensus.cc:359] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } }
03:08:17.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.534260 16050 raft_consensus.cc:740] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6cad901ab52a4c7b9a90110bcabd529d, State: Initialized, Role: FOLLOWER
03:08:17.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.535257 16050 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } }
03:08:17.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.538342 16054 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } } }
03:08:17.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.539227 16054 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [sys.catalog]: This master's current role is: FOLLOWER
03:08:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.540287 16050 sys_catalog.cc:565] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [sys.catalog]: configured and running, proceeding with master startup.
03:08:17.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.593998 16065 catalog_manager.cc:1269] Loaded cluster ID: 5d811d19a7434727a35b22e3a8e4d9bd
03:08:17.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.594667 16065 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d: loading cluster ID for follower catalog manager: success
03:08:17.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.627602 16065 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d: acquiring CA information for follower catalog manager: success
03:08:17.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:08:17 dist-test-slave-7fb5 krb5kdc[14493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656097, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.62@KRBTEST.COM for kudu/127.14.36.62@KRBTEST.COM
03:08:17.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.636286 16065 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:08:17.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.660490 15266 heartbeater.cc:344] Connected to a master server at 127.14.36.62:35391
03:08:17.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.691300 16015 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "a0016e6f9c82495288bb9911310fe995" instance_seqno: 1769656075243095) as {username='kudu', principal='kudu/127.14.36.3@KRBTEST.COM'} at 127.14.36.3:42341; Asking this server to re-register.
03:08:17.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.658514 15972 master_runner.cc:186] Error getting master registration for 127.14.36.61:45213: Network error: Client connection negotiation failed: client connection to 127.14.36.61:45213: connect: Connection refused (error 111),
03:08:17.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.693830 15266 heartbeater.cc:461] Registering TS with master...
03:08:17.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.694128 15972 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:08:17.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.697301 16015 ts_manager.cc:194] Registered new tserver with Master: a0016e6f9c82495288bb9911310fe995 (127.14.36.3:34141)
03:08:17.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:17.825009 16052 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:08:17.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:17.825596 16052 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:08:17.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:17.826239 16052 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:08:17.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:17.863641 16052 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:08:17.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:17.864171 16052 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:08:17.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:17.864627 16052 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:08:17.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:17.864854 16052 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:08:17.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:17.865056 16052 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:08:17.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:17.865259 16052 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:08:17.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:17.865407 16052 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37645/jwks.json
03:08:17.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.906049 16052 master_runner.cc:386] Master server non-default flags:
03:08:17.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:17.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.36.20:43983
03:08:17.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:17.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:17.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/data
03:08:17.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/wal
03:08:17.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:08:17.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:17.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.36.62:35391,127.14.36.61:45213,127.14.36.60:45755
03:08:17.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab
03:08:17.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.36.61
03:08:17.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:17.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:17.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:17.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.36.61:45213
03:08:17.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:17.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:08:17.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:17.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37645/jwks.json
03:08:17.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:08:17.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:08:17.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:08:17.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:17.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:17.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/data/info.pb
03:08:17.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.36.61
03:08:17.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44571
03:08:17.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:08:17.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:17.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:08:17.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:08:17.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:17.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/logs
03:08:17.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:17.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:08:17.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:08:17.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:08:17.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:08:17.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
03:08:17.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:08:17.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Jan 2026 02:43:21 UTC on 5fd53c4cbb9d
03:08:17.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10241
03:08:17.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:08:17.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.907519 16052 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:08:17.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.908710 16052 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:08:17.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:17.921586 16078 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:17.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.967550 14969 heartbeater.cc:344] Connected to a master server at 127.14.36.62:35391
03:08:17.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.971446 16015 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "1e74ae69837f4a19929b3bf70017a23d" instance_seqno: 1769656070527044) as {username='kudu', principal='kudu/127.14.36.1@KRBTEST.COM'} at 127.14.36.1:54871; Asking this server to re-register.
03:08:17.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.973625 14969 heartbeater.cc:461] Registering TS with master...
03:08:17.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:17.977972 16015 ts_manager.cc:194] Registered new tserver with Master: 1e74ae69837f4a19929b3bf70017a23d (127.14.36.1:45385)
03:08:18.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:18.237743 15119 heartbeater.cc:344] Connected to a master server at 127.14.36.62:35391
03:08:18.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:18.240918 16015 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "2bb5a457deff4c0ba8fcedcccfb2b50b" instance_seqno: 1769656072795400) as {username='kudu', principal='kudu/127.14.36.2@KRBTEST.COM'} at 127.14.36.2:37901; Asking this server to re-register.
03:08:18.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:18.242836 15119 heartbeater.cc:461] Registering TS with master...
03:08:18.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:18.246654 16015 ts_manager.cc:194] Registered new tserver with Master: 2bb5a457deff4c0ba8fcedcccfb2b50b (127.14.36.2:37651)
03:08:18.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:18.711740 16088 raft_consensus.cc:493] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:08:18.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:18.712807 16088 raft_consensus.cc:515] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } }
03:08:18.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:18.747328 16088 leader_election.cc:290] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers de1dd59a869a46bdb0165c8963557689 (127.14.36.61:45213), a5feff59896e4438ac3b32435885f30c (127.14.36.60:45755)
03:08:18.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:18.778067 16002 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.14.36.61:45213: connect: Connection refused (error 111)
03:08:18.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:18.801952 16002 leader_election.cc:336] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer de1dd59a869a46bdb0165c8963557689 (127.14.36.61:45213): Network error: Client connection negotiation failed: client connection to 127.14.36.61:45213: connect: Connection refused (error 111)
03:08:18.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:18.802955 16002 leader_election.cc:336] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer a5feff59896e4438ac3b32435885f30c (127.14.36.60:45755): Network error: Client connection negotiation failed: client connection to 127.14.36.60:45755: connect: Connection refused (error 111)
03:08:18.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:18.803509 16002 leader_election.cc:304] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6cad901ab52a4c7b9a90110bcabd529d; no voters: a5feff59896e4438ac3b32435885f30c, de1dd59a869a46bdb0165c8963557689
03:08:18.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:18.807643 16088 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
03:08:20.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:19.327689 16077 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 16052
03:08:20.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:20.082969 16077 kernel_stack_watchdog.cc:198] Thread 16052 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 402ms:
03:08:20.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
03:08:20.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
03:08:20.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
03:08:20.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
03:08:20.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:20.083844 16052 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.161s user 0.000s sys 0.002s
03:08:20.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:20.084262 16052 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.161s user 0.000s sys 0.002s
03:08:20.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:17.923300 16079 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:20.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:20.084753 16080 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 2160 milliseconds
03:08:20.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:20.086086 16081 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:20.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.086030 16052 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:08:20.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.100673 16052 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:08:20.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.104337 16052 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:08:20.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.105908 16052 hybrid_clock.cc:648] HybridClock initialized: now 1769656100105741 us; error 194 us; skew 500 ppm
03:08:20.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.107103 16052 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:08:20.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:08:20 dist-test-slave-7fb5 krb5kdc[14493](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656100, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:20.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.114794 16052 init.cc:377] Logged in from keytab as kudu/127.14.36.61@KRBTEST.COM (short username kudu)
03:08:20.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.120931 16052 webserver.cc:492] Webserver started at http://127.14.36.61:44571/ using document root <none> and password file <none>
03:08:20.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.122900 16052 fs_manager.cc:362] Metadata directory not provided
03:08:20.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.123145 16052 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:08:20.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.132562 16052 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.003s sys 0.004s
03:08:20.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.138234 16102 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:08:20.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.139710 16052 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.001s sys 0.004s
03:08:20.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.140136 16052 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/data,/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/wal
03:08:20.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "de1dd59a869a46bdb0165c8963557689"
03:08:20.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-29 03:07:43 on dist-test-slave-7fb5"
03:08:20.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.142902 16052 fs_report.cc:389] FS layout report
03:08:20.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:08:20.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/wal
03:08:20.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/wal
03:08:20.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/data/data
03:08:20.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:08:20.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:08:20.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:08:20.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:08:20.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:08:20.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:08:20.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:08:20.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:08:20.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:08:20.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:08:20.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:08:20.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.212015 16052 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:08:20.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.219375 16052 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:08:20.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.219926 16052 kserver.cc:163] Server-wide thread pool size limit: 3276
03:08:20.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:08:20 dist-test-slave-7fb5 krb5kdc[14493](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656100, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.1@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:20.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.249418 14848 init.cc:295] Successfully reacquired a new kerberos TGT
03:08:20.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.345001 16052 rpc_server.cc:307] RPC server started. Bound to: 127.14.36.61:45213
03:08:20.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.345192 16155 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.14.36.61:45213 every 8 connection(s)
03:08:20.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.348811 16052 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-1/data/info.pb
03:08:20.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.356767 14480 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu as pid 16052
03:08:20.357 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:08:20.358 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.14.36.60:45755
03:08:20.358 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
03:08:20.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.360368 14480 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu
03:08:20.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu
03:08:20.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/wal
03:08:20.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/data
03:08:20.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:08:20.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:08:20.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:08:20.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:20.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:20.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:20.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/logs
03:08:20.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/data/info.pb
03:08:20.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:20.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:08:20.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:08:20.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:08:20.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:08:20.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:20.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:20.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:20.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:08:20.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:08:20.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:20.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:20.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.36.60:45755
03:08:20.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.36.60
03:08:20.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41699
03:08:20.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab
03:08:20.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.36.60
03:08:20.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:08:20.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:08:20.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:08:20.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:08:20.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.36.20:43983
03:08:20.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:20.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:20.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:20.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:20.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.36.62:35391,127.14.36.61:45213,127.14.36.60:45755
03:08:20.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:20.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37645/jwks.json
03:08:20.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:08:20.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:08:20.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.378751 16156 sys_catalog.cc:263] Verifying existing consensus state
03:08:20.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.384824 16156 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689: Bootstrap starting.
03:08:20.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.433305 16156 log.cc:826] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689: Log is configured to *not* fsync() on all Append() calls
03:08:20.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.456712 16160 raft_consensus.cc:493] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:08:20.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.457198 16160 raft_consensus.cc:515] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } }
03:08:20.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.467849 16160 leader_election.cc:290] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers de1dd59a869a46bdb0165c8963557689 (127.14.36.61:45213), a5feff59896e4438ac3b32435885f30c (127.14.36.60:45755)
03:08:20.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.483485 16156 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:08:20.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.485237 16156 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689: Bootstrap complete.
03:08:20.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:20.497160 16002 leader_election.cc:336] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer a5feff59896e4438ac3b32435885f30c (127.14.36.60:45755): Network error: Client connection negotiation failed: client connection to 127.14.36.60:45755: connect: Connection refused (error 111)
03:08:20.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.532464 16156 raft_consensus.cc:359] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } }
03:08:20.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.536940 16156 raft_consensus.cc:740] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: de1dd59a869a46bdb0165c8963557689, State: Initialized, Role: FOLLOWER
03:08:20.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.538415 16156 consensus_queue.cc:260] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } }
03:08:20.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:08:20 dist-test-slave-7fb5 krb5kdc[14493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656097, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.62@KRBTEST.COM for kudu/127.14.36.61@KRBTEST.COM
03:08:20.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.644933 16131 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6cad901ab52a4c7b9a90110bcabd529d" candidate_term: 4 candidate_status { last_received { term: 3 index: 6 } } ignore_live_leader: false dest_uuid: "de1dd59a869a46bdb0165c8963557689" is_pre_election: true
03:08:20.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.645804 16131 raft_consensus.cc:2468] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6cad901ab52a4c7b9a90110bcabd529d in term 3.
03:08:20.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.641117 16156 sys_catalog.cc:565] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [sys.catalog]: configured and running, proceeding with master startup.
03:08:20.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.647718 16002 leader_election.cc:304] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6cad901ab52a4c7b9a90110bcabd529d, de1dd59a869a46bdb0165c8963557689; no voters: a5feff59896e4438ac3b32435885f30c
03:08:20.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.648718 16165 sys_catalog.cc:455] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } } }
03:08:20.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.649247 16160 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [term 3 FOLLOWER]: Leader pre-election won for term 4
03:08:20.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.649523 16165 sys_catalog.cc:458] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [sys.catalog]: This master's current role is: FOLLOWER
03:08:20.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.649598 16160 raft_consensus.cc:493] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [term 3 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:08:20.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.649894 16160 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [term 3 FOLLOWER]: Advancing to term 4
03:08:20.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.662411 16160 raft_consensus.cc:515] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } }
03:08:20.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.665696 16131 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6cad901ab52a4c7b9a90110bcabd529d" candidate_term: 4 candidate_status { last_received { term: 3 index: 6 } } ignore_live_leader: false dest_uuid: "de1dd59a869a46bdb0165c8963557689"
03:08:20.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.666247 16131 raft_consensus.cc:3060] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [term 3 FOLLOWER]: Advancing to term 4
03:08:20.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:20.690062 16002 leader_election.cc:336] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer a5feff59896e4438ac3b32435885f30c (127.14.36.60:45755): Network error: Client connection negotiation failed: client connection to 127.14.36.60:45755: connect: Connection refused (error 111)
03:08:20.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.691594 16160 leader_election.cc:290] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [CANDIDATE]: Term 4 election: Requested vote from peers de1dd59a869a46bdb0165c8963557689 (127.14.36.61:45213), a5feff59896e4438ac3b32435885f30c (127.14.36.60:45755)
03:08:20.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.698529 16131 raft_consensus.cc:2468] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6cad901ab52a4c7b9a90110bcabd529d in term 4.
03:08:20.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.700039 16002 leader_election.cc:304] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6cad901ab52a4c7b9a90110bcabd529d, de1dd59a869a46bdb0165c8963557689; no voters: a5feff59896e4438ac3b32435885f30c
03:08:20.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.701484 16160 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [term 4 FOLLOWER]: Leader election won for term 4
03:08:20.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.718446 16160 raft_consensus.cc:697] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [term 4 LEADER]: Becoming Leader. State: Replica: 6cad901ab52a4c7b9a90110bcabd529d, State: Running, Role: LEADER
03:08:20.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.719704 16160 consensus_queue.cc:237] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } }
03:08:20.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.775619 16168 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6cad901ab52a4c7b9a90110bcabd529d. Latest consensus state: current_term: 4 leader_uuid: "6cad901ab52a4c7b9a90110bcabd529d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } } }
03:08:20.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.785641 16168 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [sys.catalog]: This master's current role is: LEADER
03:08:20.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.789214 16179 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
03:08:20.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.791303 16180 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
03:08:20.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:08:20 dist-test-slave-7fb5 krb5kdc[14493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656095, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.3@KRBTEST.COM for kudu/127.14.36.61@KRBTEST.COM
03:08:20.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.809096 16180 catalog_manager.cc:1494] Initializing Kudu cluster ID...
03:08:20.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.816191 16180 catalog_manager.cc:1269] Loaded cluster ID: 5d811d19a7434727a35b22e3a8e4d9bd
03:08:20.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.816749 16180 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
03:08:20.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.817711 15265 heartbeater.cc:344] Connected to a master server at 127.14.36.61:45213
03:08:20.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.822618 16180 catalog_manager.cc:1514] Loading token signing keys...
03:08:20.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.825783 16121 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "a0016e6f9c82495288bb9911310fe995" instance_seqno: 1769656075243095) as {username='kudu', principal='kudu/127.14.36.3@KRBTEST.COM'} at 127.14.36.3:59357; Asking this server to re-register.
03:08:20.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.827219 16180 catalog_manager.cc:6038] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d: Loaded TSK: 0
03:08:20.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.827450 15265 heartbeater.cc:461] Registering TS with master...
03:08:20.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.828559 16180 catalog_manager.cc:1524] Initializing in-progress tserver states...
03:08:20.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.832160 16121 ts_manager.cc:194] Registered new tserver with Master: a0016e6f9c82495288bb9911310fe995 (127.14.36.3:34141)
03:08:20.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:08:20 dist-test-slave-7fb5 krb5kdc[14493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656100, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.61@KRBTEST.COM for kudu/127.14.36.62@KRBTEST.COM
03:08:20.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:08:20 dist-test-slave-7fb5 krb5kdc[14493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656100, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.61@KRBTEST.COM for kudu/127.14.36.61@KRBTEST.COM
03:08:20.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.900086 16052 master_runner.cc:186] Error getting master registration for 127.14.36.60:45755: Network error: Client connection negotiation failed: client connection to 127.14.36.60:45755: connect: Connection refused (error 111),
03:08:20.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:20.927791 16052 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:08:20.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:20.940207 16158 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:08:20.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:20.941155 16158 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:08:20.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:20.942078 16158 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:08:20.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:20.977243 16158 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:08:20.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:20.977571 16158 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:08:20.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:20.977963 16158 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:08:20.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:20.978241 16158 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:08:20.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:20.978515 16158 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:08:20.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:20.978745 16158 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:08:20.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:20.978897 16158 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37645/jwks.json
03:08:21.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:21.022193 16158 master_runner.cc:386] Master server non-default flags:
03:08:21.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:21.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.36.20:43983
03:08:21.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:21.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:21.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/data
03:08:21.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/wal
03:08:21.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:08:21.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:21.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.36.62:35391,127.14.36.61:45213,127.14.36.60:45755
03:08:21.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/krb5kdc/kudu.keytab
03:08:21.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.36.60
03:08:21.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:21.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:21.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:21.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.36.60:45755
03:08:21.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:21.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:08:21.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:21.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37645/jwks.json
03:08:21.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:08:21.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:08:21.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:08:21.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:21.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:21.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/data/info.pb
03:08:21.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.36.60
03:08:21.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41699
03:08:21.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:08:21.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:21.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:08:21.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:08:21.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:21.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/logs
03:08:21.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:21.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:08:21.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:08:21.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:08:21.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:08:21.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
03:08:21.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:08:21.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Jan 2026 02:43:21 UTC on 5fd53c4cbb9d
03:08:21.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10241
03:08:21.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:08:21.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:21.023803 16158 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:08:21.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:21.025245 16158 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:08:21.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:21.037914 16188 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:21.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:21.134688 16131 raft_consensus.cc:1275] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [term 4 FOLLOWER]: Refusing update from remote peer 6cad901ab52a4c7b9a90110bcabd529d: Log matching property violated. Preceding OpId in replica: term: 3 index: 6. Preceding OpId from leader: term: 4 index: 7. (index mismatch)
03:08:21.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:21.137121 16168 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [LEADER]: Connected to new peer: Peer: permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.001s
03:08:21.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:21.154250 16165 sys_catalog.cc:455] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6cad901ab52a4c7b9a90110bcabd529d. Latest consensus state: current_term: 4 leader_uuid: "6cad901ab52a4c7b9a90110bcabd529d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } } }
03:08:21.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:21.155143 16165 sys_catalog.cc:458] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [sys.catalog]: This master's current role is: FOLLOWER
03:08:21.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:21.172170 16002 consensus_peers.cc:597] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d -> Peer a5feff59896e4438ac3b32435885f30c (127.14.36.60:45755): Couldn't send request to peer a5feff59896e4438ac3b32435885f30c. Status: Network error: Client connection negotiation failed: client connection to 127.14.36.60:45755: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
03:08:21.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:21.188187 16168 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "6cad901ab52a4c7b9a90110bcabd529d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } } }
03:08:21.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:21.189105 16168 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d [sys.catalog]: This master's current role is: LEADER
03:08:21.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:21.191919 16165 sys_catalog.cc:455] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "6cad901ab52a4c7b9a90110bcabd529d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } } }
03:08:21.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:21.192641 16165 sys_catalog.cc:458] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689 [sys.catalog]: This master's current role is: FOLLOWER
03:08:21.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:08:21 dist-test-slave-7fb5 krb5kdc[14493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656100, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.1@KRBTEST.COM for kudu/127.14.36.61@KRBTEST.COM
03:08:21.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:21.267148 14968 heartbeater.cc:344] Connected to a master server at 127.14.36.61:45213
03:08:21.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:21.271551 16121 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "1e74ae69837f4a19929b3bf70017a23d" instance_seqno: 1769656070527044) as {username='kudu', principal='kudu/127.14.36.1@KRBTEST.COM'} at 127.14.36.1:40327; Asking this server to re-register.
03:08:21.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:21.274091 14968 heartbeater.cc:461] Registering TS with master...
03:08:21.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:21.280675 16121 ts_manager.cc:194] Registered new tserver with Master: 1e74ae69837f4a19929b3bf70017a23d (127.14.36.1:45385)
03:08:21.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:08:21 dist-test-slave-7fb5 krb5kdc[14493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656092, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.2@KRBTEST.COM for kudu/127.14.36.61@KRBTEST.COM
03:08:21.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:21.471812 15118 heartbeater.cc:344] Connected to a master server at 127.14.36.61:45213
03:08:21.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:21.475346 16121 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "2bb5a457deff4c0ba8fcedcccfb2b50b" instance_seqno: 1769656072795400) as {username='kudu', principal='kudu/127.14.36.2@KRBTEST.COM'} at 127.14.36.2:52141; Asking this server to re-register.
03:08:21.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:21.477378 15118 heartbeater.cc:461] Registering TS with master...
03:08:21.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:21.481515 16121 ts_manager.cc:194] Registered new tserver with Master: 2bb5a457deff4c0ba8fcedcccfb2b50b (127.14.36.2:37651)
03:08:21.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:21.629879 15113 debug-util.cc:398] Leaking SignalData structure 0x7b08000bb960 after lost signal to thread 14982
03:08:21.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:21.631953 15113 debug-util.cc:398] Leaking SignalData structure 0x7b08000b2340 after lost signal to thread 15116
03:08:21.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:21.766146 14963 debug-util.cc:398] Leaking SignalData structure 0x7b08000b59a0 after lost signal to thread 14834
03:08:21.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:21.767477 14963 debug-util.cc:398] Leaking SignalData structure 0x7b08000bbf60 after lost signal to thread 14966
03:08:21.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:21.816347 16179 catalog_manager.cc:1269] Loaded cluster ID: 5d811d19a7434727a35b22e3a8e4d9bd
03:08:21.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:21.817054 16179 catalog_manager.cc:1562] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689: loading cluster ID for follower catalog manager: success
03:08:21.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:21.826372 16179 catalog_manager.cc:1584] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689: acquiring CA information for follower catalog manager: success
03:08:21.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:21.833276 16179 catalog_manager.cc:1612] T 00000000000000000000000000000000 P de1dd59a869a46bdb0165c8963557689: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:08:21.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:21.993934 14969 heartbeater.cc:499] Master 127.14.36.62:35391 was elected leader, sending a full tablet report...
03:08:22.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:22.264017 15119 heartbeater.cc:499] Master 127.14.36.62:35391 was elected leader, sending a full tablet report...
03:08:22.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:08:22 dist-test-slave-7fb5 krb5kdc[14493](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656102, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.2@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:22.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:22.596639 14998 init.cc:295] Successfully reacquired a new kerberos TGT
03:08:22.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:22.710788 15972 master_runner.cc:186] Error getting master registration for 127.14.36.60:45755: Network error: Client connection negotiation failed: client connection to 127.14.36.60:45755: connect: Connection refused (error 111),
03:08:22.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:22.711438 15972 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:08:22.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:22.794476 15266 heartbeater.cc:499] Master 127.14.36.62:35391 was elected leader, sending a full tablet report...
03:08:23.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:23.665097 16002 consensus_peers.cc:597] T 00000000000000000000000000000000 P 6cad901ab52a4c7b9a90110bcabd529d -> Peer a5feff59896e4438ac3b32435885f30c (127.14.36.60:45755): Couldn't send request to peer a5feff59896e4438ac3b32435885f30c. Status: Network error: Client connection negotiation failed: client connection to 127.14.36.60:45755: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
03:08:24.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:24.255035 16002 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.14.36.60:45755: connect: Connection refused (error 111) [suppressed 9 similar messages]
03:08:25.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:08:25 dist-test-slave-7fb5 krb5kdc[14493](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656105, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.3@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:25.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:25.034706 15147 init.cc:295] Successfully reacquired a new kerberos TGT
03:08:25.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:22.442600 16187 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 16158
03:08:25.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:25.132382 16187 kernel_stack_watchdog.cc:198] Thread 16158 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 399ms:
03:08:25.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
03:08:25.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
03:08:25.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
03:08:25.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
03:08:25.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:25.132988 16158 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 4.094s user 0.919s sys 1.637s
03:08:25.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:25.133352 16158 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 4.094s user 0.919s sys 1.637s
03:08:25.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:21.040346 16189 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:25.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:25.135182 16191 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:25.139933 16190 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 4100 milliseconds
03:08:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:25.140574 16158 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:08:25.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:25.142287 16158 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:08:25.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:25.153036 16158 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:08:25.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:25.154565 16158 hybrid_clock.cc:648] HybridClock initialized: now 1769656105154519 us; error 53 us; skew 500 ppm
03:08:25.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:25.155817 16158 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:08:25.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:08:25 dist-test-slave-7fb5 krb5kdc[14493](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656105, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:25.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:25.164541 16158 init.cc:377] Logged in from keytab as kudu/127.14.36.60@KRBTEST.COM (short username kudu)
03:08:25.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:25.216698 16158 webserver.cc:492] Webserver started at http://127.14.36.60:41699/ using document root <none> and password file <none>
03:08:25.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:25.218400 16158 fs_manager.cc:362] Metadata directory not provided
03:08:25.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:25.218899 16158 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:08:25.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:25.251403 16158 fs_manager.cc:714] Time spent opening directory manager: real 0.015s user 0.005s sys 0.001s
03:08:25.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:25.267395 16234 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:08:25.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:25.269279 16158 fs_manager.cc:730] Time spent opening block manager: real 0.014s user 0.005s sys 0.005s
03:08:25.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:25.269881 16158 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/data,/tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/wal
03:08:25.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a5feff59896e4438ac3b32435885f30c"
03:08:25.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-29 03:07:45 on dist-test-slave-7fb5"
03:08:25.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:25.273064 16158 fs_report.cc:389] FS layout report
03:08:25.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:08:25.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/wal
03:08:25.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/wal
03:08:25.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/data/data
03:08:25.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:08:25.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:08:25.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:08:25.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:08:25.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:08:25.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:08:25.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:08:25.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:08:25.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:08:25.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:08:25.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:08:25.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:25.444993 16158 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:08:25.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:25.471103 16158 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:08:25.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:25.472044 16158 kserver.cc:163] Server-wide thread pool size limit: 3276
03:08:25.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:25.881253 16295 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.14.36.60:45755 every 8 connection(s)
03:08:25.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:25.882207 16158 rpc_server.cc:307] RPC server started. Bound to: 127.14.36.60:45755
03:08:25.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:25.886972 16158 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskuwp0L7/test-tmp/mini-kudu-cluster5058322186600518696/master-2/data/info.pb
03:08:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:25.896790 14480 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu as pid 16158
03:08:25.897 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:08:25.898 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.14.36.62:35391(127.14.36.62:35391)] connecting to peer
03:08:25.904 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.14.36.62:35391(127.14.36.62:35391)] Successfully connected to peer
03:08:25.929 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.14.36.62:35391(127.14.36.62:35391)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
03:08:25.946 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.14.36.62:35391(127.14.36.62:35391)] handling channelInactive
03:08:25.951 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.14.36.62:35391(127.14.36.62:35391)] disconnecting while in state TERMINATED
03:08:25.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:25.948393 16296 sys_catalog.cc:263] Verifying existing consensus state
03:08:25.954 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.14.36.61:45213(127.14.36.61:45213)] disconnecting while in state TERMINATED
03:08:25.957 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.14.36.60:45755(127.14.36.60:45755)] disconnecting while in state TERMINATED
03:08:25.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:25.967079 16296 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c: Bootstrap starting.
03:08:25.978 [DEBUG - Time-limited test] (AsyncKuduClient.java:2704) Releasing all remaining resources
03:08:25.993 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
03:08:25.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:25.998732 16297 negotiation.cc:337] Failed RPC negotiation. Trace:
03:08:25.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0129 03:08:25.903738 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:40638 (local address 127.14.36.62:35391)
03:08:25.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0129 03:08:25.926811 (+ 23073us) server_negotiation.cc:207] Beginning negotiation
03:08:25.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0129 03:08:25.926846 (+ 35us) server_negotiation.cc:400] Waiting for connection header
03:08:25.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0129 03:08:25.926925 (+ 79us) server_negotiation.cc:408] Connection header received
03:08:25.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0129 03:08:25.927169 (+ 244us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
03:08:25.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0129 03:08:25.927207 (+ 38us) server_negotiation.cc:462] Received NEGOTIATE request from client
03:08:25.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0129 03:08:25.927487 (+ 280us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
03:08:25.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0129 03:08:25.982458 (+ 54971us) server_negotiation.cc:227] Negotiated authn=SASL
03:08:25.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0129 03:08:25.983256 (+ 798us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:40638: BlockingRecv error: recv got EOF from 127.0.0.1:40638 (error 108)
03:08:25.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":22386,"thread_start_us":24138,"threads_started":1}
03:08:26.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:26.012565 14480 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu with pid 14830
03:08:26.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:26.082644 14480 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu with pid 14981
03:08:26.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 29 03:08:26 dist-test-slave-7fb5 krb5kdc[14493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769656100, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.36.61@KRBTEST.COM for kudu/127.14.36.60@KRBTEST.COM
03:08:26.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:26.100282 16296 log.cc:826] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c: Log is configured to *not* fsync() on all Append() calls
03:08:26.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:26.111595 16052 master_runner.cc:186] Error getting master registration for 127.14.36.60:45755: OK, instance_id { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" instance_seqno: 1769656105746077 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:08:26.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:26.112386 16052 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:08:26.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:26.121255 14480 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu with pid 15131
03:08:26.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:26.133422 16296 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:08:26.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:26.134194 16296 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c: Bootstrap complete.
03:08:26.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:26.161167 14480 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu with pid 15972
03:08:26.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:26.165171 16296 raft_consensus.cc:359] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } }
03:08:26.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:26.168874 16296 raft_consensus.cc:740] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: a5feff59896e4438ac3b32435885f30c, State: Initialized, Role: FOLLOWER
03:08:26.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:26.169896 16296 consensus_queue.cc:260] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } }
03:08:26.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:26.185034 16306 sys_catalog.cc:455] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6cad901ab52a4c7b9a90110bcabd529d" member_type: VOTER last_known_addr { host: "127.14.36.62" port: 35391 } } peers { permanent_uuid: "de1dd59a869a46bdb0165c8963557689" member_type: VOTER last_known_addr { host: "127.14.36.61" port: 45213 } } peers { permanent_uuid: "a5feff59896e4438ac3b32435885f30c" member_type: VOTER last_known_addr { host: "127.14.36.60" port: 45755 } } }
03:08:26.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:26.187083 16296 sys_catalog.cc:565] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c [sys.catalog]: configured and running, proceeding with master startup.
03:08:26.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:26.188143 16306 sys_catalog.cc:458] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c [sys.catalog]: This master's current role is: FOLLOWER
03:08:26.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:26.205888 14480 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu with pid 16052
03:08:26.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:26.244962 16317 catalog_manager.cc:1269] Loaded cluster ID: 5d811d19a7434727a35b22e3a8e4d9bd
03:08:26.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:26.245257 16317 catalog_manager.cc:1562] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c: loading cluster ID for follower catalog manager: success
03:08:26.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:26.254551 16317 catalog_manager.cc:1584] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c: acquiring CA information for follower catalog manager: success
03:08:26.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:26.260102 16317 catalog_manager.cc:1612] T 00000000000000000000000000000000 P a5feff59896e4438ac3b32435885f30c: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:08:26.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:26.264205 14480 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu with pid 16158
03:08:27.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-01-29T03:08:27Z chronyd exiting
03:08:28.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:28.239696 14181 debug-util.cc:398] Leaking SignalData structure 0x7b08000baa20 after lost signal to thread 14036
03:08:28.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260129 03:08:28.240624 14181 debug-util.cc:398] Leaking SignalData structure 0x7b08000b6740 after lost signal to thread 14184
03:08:28.538 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
03:08:28.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:28.539842 13750 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu with pid 14033
03:08:28.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:28.576308 13750 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu with pid 14192
03:08:28.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:28.605733 13750 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu with pid 14337
03:08:28.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:28.632160 13750 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu with pid 13762
03:08:28.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:28.662822 13750 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu with pid 13833
03:08:28.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260129 03:08:28.693310 13750 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskuwp0L7/build/tsan/bin/kudu with pid 13908
03:08:29.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-01-29T03:08:29Z chronyd exiting
03:08:30.885 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565) ~[main/:?]
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
03:08:31.358 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
03:08:31.358 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565) ~[main/:?]
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more