14:14:12.594 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
14:14:12.594 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskB9qii1/test-tmp
14:14:12.594 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
14:14:12.594 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskB9qii1/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
14:14:12.595 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
14:14:12.902 [DEBUG - main] (MiniKuduCluster.java:195) Response:
14:14:12.902 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
14:14:12.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:12.905694 20207 mini_oidc.cc:150] Starting JWKS server
14:14:12.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:12.908730 20207 webserver.cc:492] Webserver started at http://127.0.0.1:43923/ using document root <none> and password file <none>
14:14:12.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:12.909176 20207 mini_oidc.cc:186] Starting OIDC Discovery server
14:14:12.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:12.910843 20207 webserver.cc:492] Webserver started at http://127.0.0.1:42009/ using document root <none> and password file <none>
14:14:12.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-25T14:14:12Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
14:14:12.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-25T14:14:12Z Disabled control of system clock
14:14:12.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:12.968019 20207 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu
14:14:12.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu
14:14:12.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-0/wal
14:14:12.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-0/data
14:14:12.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:14:12.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:14:12.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:14:12.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:12.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:12.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:12.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-0/logs
14:14:12.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-0/data/info.pb
14:14:12.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:12.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:14:12.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:14:12.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:14:12.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:14:12.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:12.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:12.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:12.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:14:12.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:14:12.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:14:12.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:14:12.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.187.254:42383
14:14:12.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.187.254
14:14:12.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:14:12.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.187.212:40939
14:14:12.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:12.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:12.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:14:12.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:14:12.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.187.254:42383,127.19.187.253:36413,127.19.187.252:37535
14:14:12.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:14:12.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43923/jwks.json with env {}
14:14:13.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:13.247819 20219 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:14:13.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:13.248350 20219 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:14:13.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:13.248821 20219 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:14:13.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:13.278604 20219 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:14:13.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:13.278854 20219 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:14:13.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:13.279047 20219 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:14:13.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:13.279263 20219 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:14:13.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:13.279500 20219 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:14:13.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:13.279652 20219 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43923/jwks.json
14:14:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:13.313617 20219 master_runner.cc:386] Master server non-default flags:
14:14:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.187.212:40939
14:14:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:14:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-0/data
14:14:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-0/wal
14:14:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:14:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.187.254:42383,127.19.187.253:36413,127.19.187.252:37535
14:14:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:14:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.187.254:42383
14:14:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:14:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:14:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:14:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43923/jwks.json
14:14:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-0/data/info.pb
14:14:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.187.254
14:14:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:14:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:14:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:14:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:14:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-0/logs
14:14:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:14:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:14:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:14:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:14:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d0cb013a5aad754f2ed06c25f5ec3d8a1873622
14:14:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:14:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Oct 2025 13:43:23 UTC on 5fd53c4cbb9d
14:14:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8713
14:14:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:14:13.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:13.314978 20219 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:14:13.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:13.316597 20219 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:14:13.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:13.326390 20225 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:14.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:13.328275 20226 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:14.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:14.596417 20219 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.269s user 0.001s sys 0.004s
14:14:14.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:14.596817 20219 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.269s user 0.001s sys 0.004s
14:14:14.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:14.597637 20227 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1264 milliseconds
14:14:14.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:14.598330 20228 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:14.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:14.598381 20219 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:14:14.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:14.599804 20219 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:14:14.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:14.602275 20219 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:14:14.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:14.603598 20219 hybrid_clock.cc:648] HybridClock initialized: now 1761401654603571 us; error 37 us; skew 500 ppm
14:14:14.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:14.604436 20219 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:14:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:14.611493 20219 webserver.cc:492] Webserver started at http://127.19.187.254:43349/ using document root <none> and password file <none>
14:14:14.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:14.612465 20219 fs_manager.cc:362] Metadata directory not provided
14:14:14.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:14.612690 20219 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:14:14.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:14.613202 20219 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:14:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:14.617637 20219 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-0/data/instance:
14:14:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "92e8830b9a8c416497ac98afc1c5468a"
14:14:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 14:14:14 on dist-test-slave-bg5s"
14:14:14.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:14.618810 20219 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-0/wal/instance:
14:14:14.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "92e8830b9a8c416497ac98afc1c5468a"
14:14:14.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 14:14:14 on dist-test-slave-bg5s"
14:14:14.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:14.626659 20219 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.000s
14:14:14.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:14.632814 20235 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:14:14.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:14.634085 20219 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
14:14:14.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:14.634429 20219 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-0/data,/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-0/wal
14:14:14.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "92e8830b9a8c416497ac98afc1c5468a"
14:14:14.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 14:14:14 on dist-test-slave-bg5s"
14:14:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:14.634758 20219 fs_report.cc:389] FS layout report
14:14:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:14:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-0/wal
14:14:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-0/wal
14:14:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-0/data/data
14:14:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:14:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:14:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:14:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:14:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:14:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:14:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:14:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:14:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:14:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:14:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:14:14.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:14.698261 20219 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:14:14.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:14.699728 20219 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:14:14.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:14.700176 20219 kserver.cc:163] Server-wide thread pool size limit: 3276
14:14:14.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:14.768762 20219 rpc_server.cc:307] RPC server started. Bound to: 127.19.187.254:42383
14:14:14.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:14.768839 20286 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.187.254:42383 every 8 connection(s)
14:14:14.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:14.771322 20219 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-0/data/info.pb
14:14:14.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:14.777050 20287 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:14:14.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:14.779767 20207 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu as pid 20219
14:14:14.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:14.780165 20207 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-0/wal/instance
14:14:14.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:14.783232 20207 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu
14:14:14.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu
14:14:14.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-1/wal
14:14:14.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-1/data
14:14:14.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:14:14.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:14:14.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:14:14.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:14.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:14.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:14.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-1/logs
14:14:14.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-1/data/info.pb
14:14:14.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:14.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:14:14.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:14:14.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:14:14.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:14:14.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:14.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:14.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:14.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:14:14.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:14:14.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:14:14.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:14:14.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.187.253:36413
14:14:14.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.187.253
14:14:14.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:14:14.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.187.212:40939
14:14:14.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:14.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:14.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:14:14.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:14:14.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.187.254:42383,127.19.187.253:36413,127.19.187.252:37535
14:14:14.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:14:14.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43923/jwks.json with env {}
14:14:14.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:14.787170 20287 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.187.254" port: 42383 } has no permanent_uuid. Determining permanent_uuid...
14:14:14.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:14.816104 20287 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.187.253" port: 36413 } has no permanent_uuid. Determining permanent_uuid...
14:14:14.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:14.818267 20237 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.187.253:36413: connect: Connection refused (error 111)
14:14:14.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:14.822037 20287 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.187.253:36413: Network error: Client connection negotiation failed: client connection to 127.19.187.253:36413: connect: Connection refused (error 111)
14:14:14.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:14.871610 20287 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.187.253" port: 36413 } attempt: 1
14:14:14.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:14.875411 20287 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.187.253:36413: Network error: Client connection negotiation failed: client connection to 127.19.187.253:36413: connect: Connection refused (error 111)
14:14:14.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:14.943940 20287 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.187.253" port: 36413 } attempt: 2
14:14:14.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:14.947885 20287 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.187.253:36413: Network error: Client connection negotiation failed: client connection to 127.19.187.253:36413: connect: Connection refused (error 111)
14:14:15.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:15.039260 20287 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.187.253" port: 36413 } attempt: 3
14:14:15.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:15.043421 20287 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.187.253:36413: Network error: Client connection negotiation failed: client connection to 127.19.187.253:36413: connect: Connection refused (error 111)
14:14:15.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:15.075806 20289 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:14:15.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:15.076406 20289 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:14:15.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:15.076958 20289 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:14:15.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:15.106798 20289 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:14:15.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:15.107050 20289 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:14:15.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:15.107223 20289 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:14:15.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:15.107411 20289 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:14:15.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:15.107609 20289 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:14:15.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:15.107753 20289 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43923/jwks.json
14:14:15.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:15.140817 20289 master_runner.cc:386] Master server non-default flags:
14:14:15.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:15.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.187.212:40939
14:14:15.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:15.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:14:15.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-1/data
14:14:15.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-1/wal
14:14:15.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:14:15.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.187.254:42383,127.19.187.253:36413,127.19.187.252:37535
14:14:15.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:15.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:15.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:14:15.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.187.253:36413
14:14:15.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:14:15.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:14:15.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:14:15.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43923/jwks.json
14:14:15.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:15.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:15.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-1/data/info.pb
14:14:15.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.187.253
14:14:15.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:14:15.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:14:15.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:15.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:14:15.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:14:15.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:15.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-1/logs
14:14:15.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:15.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:14:15.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:14:15.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:14:15.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:14:15.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d0cb013a5aad754f2ed06c25f5ec3d8a1873622
14:14:15.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:14:15.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Oct 2025 13:43:23 UTC on 5fd53c4cbb9d
14:14:15.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8713
14:14:15.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:14:15.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:15.142092 20289 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:14:15.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:15.143539 20289 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:14:15.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:15.153601 20298 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:15.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:15.186899 20287 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.187.253" port: 36413 } attempt: 4
14:14:15.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:15.194649 20287 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.187.253:36413: Network error: Client connection negotiation failed: client connection to 127.19.187.253:36413: connect: Connection refused (error 111)
14:14:15.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:15.494208 20287 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.187.253" port: 36413 } attempt: 5
14:14:15.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:15.503355 20287 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.187.253:36413: Network error: Client connection negotiation failed: client connection to 127.19.187.253:36413: connect: Connection refused (error 111)
14:14:16.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:16.050982 20287 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.187.253" port: 36413 } attempt: 6
14:14:16.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:16.078616 20287 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.187.253:36413: Network error: Client connection negotiation failed: client connection to 127.19.187.253:36413: connect: Connection refused (error 111)
14:14:16.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:15.153960 20299 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:16.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:16.339862 20301 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:16.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:16.342427 20300 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1184 milliseconds
14:14:16.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:16.343145 20289 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.189s user 0.403s sys 0.777s
14:14:16.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:16.343430 20289 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.190s user 0.403s sys 0.781s
14:14:16.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:16.343684 20289 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:14:16.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:16.344758 20289 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:14:16.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:16.350643 20289 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:14:16.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:16.351981 20289 hybrid_clock.cc:648] HybridClock initialized: now 1761401656351947 us; error 34 us; skew 500 ppm
14:14:16.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:16.352768 20289 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:14:16.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:16.359726 20289 webserver.cc:492] Webserver started at http://127.19.187.253:35661/ using document root <none> and password file <none>
14:14:16.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:16.360589 20289 fs_manager.cc:362] Metadata directory not provided
14:14:16.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:16.360805 20289 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:14:16.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:16.361234 20289 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:14:16.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:16.365509 20289 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-1/data/instance:
14:14:16.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "05624e87b9a943dcb29bebb3750314e5"
14:14:16.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 14:14:16 on dist-test-slave-bg5s"
14:14:16.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:16.366591 20289 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-1/wal/instance:
14:14:16.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "05624e87b9a943dcb29bebb3750314e5"
14:14:16.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 14:14:16 on dist-test-slave-bg5s"
14:14:16.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:16.374245 20289 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.009s sys 0.001s
14:14:16.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:16.380362 20310 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:14:16.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:16.381429 20289 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
14:14:16.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:16.381718 20289 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-1/data,/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-1/wal
14:14:16.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "05624e87b9a943dcb29bebb3750314e5"
14:14:16.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 14:14:16 on dist-test-slave-bg5s"
14:14:16.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:16.382018 20289 fs_report.cc:389] FS layout report
14:14:16.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:14:16.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-1/wal
14:14:16.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-1/wal
14:14:16.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-1/data/data
14:14:16.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:14:16.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:14:16.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:14:16.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:14:16.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:14:16.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:14:16.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:14:16.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:14:16.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:14:16.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:14:16.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:14:16.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:16.449934 20289 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:14:16.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:16.451328 20289 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:14:16.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:16.451721 20289 kserver.cc:163] Server-wide thread pool size limit: 3276
14:14:16.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:16.518791 20289 rpc_server.cc:307] RPC server started. Bound to: 127.19.187.253:36413
14:14:16.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:16.518886 20361 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.187.253:36413 every 8 connection(s)
14:14:16.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:16.521368 20289 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-1/data/info.pb
14:14:16.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:16.525405 20207 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu as pid 20289
14:14:16.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:16.525858 20207 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-1/wal/instance
14:14:16.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:16.527475 20362 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:14:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:16.528779 20207 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu
14:14:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu
14:14:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-2/wal
14:14:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-2/data
14:14:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:14:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:14:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:14:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-2/logs
14:14:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-2/data/info.pb
14:14:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:14:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:14:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:14:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:14:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:14:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:14:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:14:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:14:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.187.252:37535
14:14:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.187.252
14:14:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:14:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.187.212:40939
14:14:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:14:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:14:16.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.187.254:42383,127.19.187.253:36413,127.19.187.252:37535
14:14:16.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:14:16.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43923/jwks.json with env {}
14:14:16.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:16.540336 20362 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.187.254" port: 42383 } has no permanent_uuid. Determining permanent_uuid...
14:14:16.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:16.559883 20362 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.187.253" port: 36413 } has no permanent_uuid. Determining permanent_uuid...
14:14:16.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:16.574173 20362 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.187.252" port: 37535 } has no permanent_uuid. Determining permanent_uuid...
14:14:16.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:16.576511 20311 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.187.252:37535: connect: Connection refused (error 111)
14:14:16.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:16.580679 20362 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.187.252:37535: Network error: Client connection negotiation failed: client connection to 127.19.187.252:37535: connect: Connection refused (error 111)
14:14:16.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:16.630128 20362 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.187.252" port: 37535 } attempt: 1
14:14:16.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:16.633701 20362 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.187.252:37535: Network error: Client connection negotiation failed: client connection to 127.19.187.252:37535: connect: Connection refused (error 111)
14:14:16.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:16.702203 20362 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.187.252" port: 37535 } attempt: 2
14:14:16.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:16.705821 20362 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.187.252:37535: Network error: Client connection negotiation failed: client connection to 127.19.187.252:37535: connect: Connection refused (error 111)
14:14:16.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:16.797245 20362 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.187.252" port: 37535 } attempt: 3
14:14:16.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:16.800987 20362 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.187.252:37535: Network error: Client connection negotiation failed: client connection to 127.19.187.252:37535: connect: Connection refused (error 111)
14:14:16.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:16.821239 20364 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:14:16.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:16.821743 20364 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:14:16.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:16.822211 20364 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:14:16.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:16.852810 20364 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:14:16.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:16.853049 20364 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:14:16.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:16.853224 20364 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:14:16.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:16.853397 20364 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:14:16.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:16.853565 20364 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:14:16.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:16.853693 20364 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43923/jwks.json
14:14:16.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:16.886586 20364 master_runner.cc:386] Master server non-default flags:
14:14:16.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:16.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.187.212:40939
14:14:16.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:16.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:14:16.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-2/data
14:14:16.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-2/wal
14:14:16.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:14:16.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.187.254:42383,127.19.187.253:36413,127.19.187.252:37535
14:14:16.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:16.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:16.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:14:16.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.187.252:37535
14:14:16.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:14:16.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:14:16.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:14:16.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43923/jwks.json
14:14:16.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:16.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:16.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-2/data/info.pb
14:14:16.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.187.252
14:14:16.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:14:16.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:14:16.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:16.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:14:16.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:14:16.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:16.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-2/logs
14:14:16.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:16.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:14:16.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:14:16.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:14:16.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:14:16.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d0cb013a5aad754f2ed06c25f5ec3d8a1873622
14:14:16.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:14:16.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Oct 2025 13:43:23 UTC on 5fd53c4cbb9d
14:14:16.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8713
14:14:16.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:14:16.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:16.887845 20364 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:14:16.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:16.889318 20364 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:14:16.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:16.900000 20374 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:16.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:16.944355 20362 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.187.252" port: 37535 } attempt: 4
14:14:16.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:16.951586 20362 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.187.252:37535: Network error: Client connection negotiation failed: client connection to 127.19.187.252:37535: connect: Connection refused (error 111)
14:14:17.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:17.139238 20287 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.187.253" port: 36413 } attempt: 7
14:14:17.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:17.184530 20287 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.187.252" port: 37535 } has no permanent_uuid. Determining permanent_uuid...
14:14:17.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:17.193066 20287 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.187.252:37535: Network error: Client connection negotiation failed: client connection to 127.19.187.252:37535: connect: Connection refused (error 111)
14:14:17.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:17.251120 20362 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.187.252" port: 37535 } attempt: 5
14:14:17.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:17.254725 20287 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.187.252" port: 37535 } attempt: 1
14:14:17.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:17.259625 20362 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.187.252:37535: Network error: Client connection negotiation failed: client connection to 127.19.187.252:37535: connect: Connection refused (error 111)
14:14:17.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:17.265749 20287 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.187.252:37535: Network error: Client connection negotiation failed: client connection to 127.19.187.252:37535: connect: Connection refused (error 111)
14:14:17.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:17.347247 20287 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.187.252" port: 37535 } attempt: 2
14:14:17.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:17.354429 20287 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.187.252:37535: Network error: Client connection negotiation failed: client connection to 127.19.187.252:37535: connect: Connection refused (error 111)
14:14:17.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:17.439944 20287 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.187.252" port: 37535 } attempt: 3
14:14:17.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:17.450973 20287 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.187.252:37535: Network error: Client connection negotiation failed: client connection to 127.19.187.252:37535: connect: Connection refused (error 111)
14:14:17.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:17.591482 20287 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.187.252" port: 37535 } attempt: 4
14:14:17.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:17.598834 20287 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.187.252:37535: Network error: Client connection negotiation failed: client connection to 127.19.187.252:37535: connect: Connection refused (error 111)
14:14:17.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:17.807126 20362 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.187.252" port: 37535 } attempt: 6
14:14:17.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:17.829761 20362 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.187.252:37535: Network error: Client connection negotiation failed: client connection to 127.19.187.252:37535: connect: Connection refused (error 111)
14:14:17.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:17.882395 20287 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.187.252" port: 37535 } attempt: 5
14:14:17.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:17.888962 20287 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.187.252:37535: Network error: Client connection negotiation failed: client connection to 127.19.187.252:37535: connect: Connection refused (error 111)
14:14:18.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.441497 20287 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.187.252" port: 37535 } attempt: 6
14:14:18.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:18.479272 20287 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.187.252:37535: Network error: Client connection negotiation failed: client connection to 127.19.187.252:37535: connect: Connection refused (error 111)
14:14:18.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:18.303117 20373 debug-util.cc:398] Leaking SignalData structure 0x7b0800037520 after lost signal to thread 20364
14:14:18.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:18.629025 20364 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.729s user 0.575s sys 1.148s
14:14:18.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:18.629419 20364 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.730s user 0.575s sys 1.148s
14:14:18.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:16.900424 20375 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:18.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:18.631150 20377 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:18.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:18.633968 20376 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1730 milliseconds
14:14:18.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.634032 20364 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:14:18.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.635031 20364 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:14:18.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.637341 20364 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:14:18.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.638644 20364 hybrid_clock.cc:648] HybridClock initialized: now 1761401658638606 us; error 45 us; skew 500 ppm
14:14:18.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.639405 20364 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:14:18.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.645399 20364 webserver.cc:492] Webserver started at http://127.19.187.252:35107/ using document root <none> and password file <none>
14:14:18.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.646356 20364 fs_manager.cc:362] Metadata directory not provided
14:14:18.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.646559 20364 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:14:18.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.646936 20364 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:14:18.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.651055 20364 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-2/data/instance:
14:14:18.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1a132e8fe7dd4ee594ce6cbf8a76b157"
14:14:18.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 14:14:18 on dist-test-slave-bg5s"
14:14:18.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.652037 20364 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-2/wal/instance:
14:14:18.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1a132e8fe7dd4ee594ce6cbf8a76b157"
14:14:18.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 14:14:18 on dist-test-slave-bg5s"
14:14:18.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.659543 20364 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.002s sys 0.006s
14:14:18.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.664831 20391 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:14:18.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.665804 20364 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.005s sys 0.000s
14:14:18.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.666087 20364 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-2/data,/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-2/wal
14:14:18.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1a132e8fe7dd4ee594ce6cbf8a76b157"
14:14:18.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 14:14:18 on dist-test-slave-bg5s"
14:14:18.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.666363 20364 fs_report.cc:389] FS layout report
14:14:18.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:14:18.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-2/wal
14:14:18.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-2/wal
14:14:18.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-2/data/data
14:14:18.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:14:18.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:14:18.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:14:18.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:14:18.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:14:18.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:14:18.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:14:18.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:14:18.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:14:18.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:14:18.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:14:18.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.726847 20364 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:14:18.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.728361 20364 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:14:18.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.728749 20364 kserver.cc:163] Server-wide thread pool size limit: 3276
14:14:18.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.799341 20364 rpc_server.cc:307] RPC server started. Bound to: 127.19.187.252:37535
14:14:18.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.799484 20442 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.187.252:37535 every 8 connection(s)
14:14:18.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.801858 20364 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-2/data/info.pb
14:14:18.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.805251 20207 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu as pid 20364
14:14:18.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.805721 20207 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/master-2/wal/instance
14:14:18.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.808532 20443 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:14:18.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.824415 20443 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.187.254" port: 42383 } has no permanent_uuid. Determining permanent_uuid...
14:14:18.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.848140 20443 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.187.253" port: 36413 } has no permanent_uuid. Determining permanent_uuid...
14:14:18.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.857594 20443 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.187.252" port: 37535 } has no permanent_uuid. Determining permanent_uuid...
14:14:18.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.872072 20443 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1a132e8fe7dd4ee594ce6cbf8a76b157: Bootstrap starting.
14:14:18.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.878031 20443 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 1a132e8fe7dd4ee594ce6cbf8a76b157: Neither blocks nor log segments found. Creating new log.
14:14:18.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.880040 20443 log.cc:826] T 00000000000000000000000000000000 P 1a132e8fe7dd4ee594ce6cbf8a76b157: Log is configured to *not* fsync() on all Append() calls
14:14:18.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.884105 20443 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1a132e8fe7dd4ee594ce6cbf8a76b157: No bootstrap required, opened a new log
14:14:18.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.890314 20362 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.187.252" port: 37535 } attempt: 7
14:14:18.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.904717 20443 raft_consensus.cc:359] T 00000000000000000000000000000000 P 1a132e8fe7dd4ee594ce6cbf8a76b157 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "92e8830b9a8c416497ac98afc1c5468a" member_type: VOTER last_known_addr { host: "127.19.187.254" port: 42383 } } peers { permanent_uuid: "05624e87b9a943dcb29bebb3750314e5" member_type: VOTER last_known_addr { host: "127.19.187.253" port: 36413 } } peers { permanent_uuid: "1a132e8fe7dd4ee594ce6cbf8a76b157" member_type: VOTER last_known_addr { host: "127.19.187.252" port: 37535 } }
14:14:18.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.905336 20443 raft_consensus.cc:385] T 00000000000000000000000000000000 P 1a132e8fe7dd4ee594ce6cbf8a76b157 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:14:18.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.905596 20443 raft_consensus.cc:740] T 00000000000000000000000000000000 P 1a132e8fe7dd4ee594ce6cbf8a76b157 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1a132e8fe7dd4ee594ce6cbf8a76b157, State: Initialized, Role: FOLLOWER
14:14:18.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.906275 20443 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1a132e8fe7dd4ee594ce6cbf8a76b157 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "92e8830b9a8c416497ac98afc1c5468a" member_type: VOTER last_known_addr { host: "127.19.187.254" port: 42383 } } peers { permanent_uuid: "05624e87b9a943dcb29bebb3750314e5" member_type: VOTER last_known_addr { host: "127.19.187.253" port: 36413 } } peers { permanent_uuid: "1a132e8fe7dd4ee594ce6cbf8a76b157" member_type: VOTER last_known_addr { host: "127.19.187.252" port: 37535 } }
14:14:18.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.906944 20362 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 05624e87b9a943dcb29bebb3750314e5: Bootstrap starting.
14:14:18.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.908764 20453 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1a132e8fe7dd4ee594ce6cbf8a76b157 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "92e8830b9a8c416497ac98afc1c5468a" member_type: VOTER last_known_addr { host: "127.19.187.254" port: 42383 } } peers { permanent_uuid: "05624e87b9a943dcb29bebb3750314e5" member_type: VOTER last_known_addr { host: "127.19.187.253" port: 36413 } } peers { permanent_uuid: "1a132e8fe7dd4ee594ce6cbf8a76b157" member_type: VOTER last_known_addr { host: "127.19.187.252" port: 37535 } } }
14:14:18.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.909626 20453 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1a132e8fe7dd4ee594ce6cbf8a76b157 [sys.catalog]: This master's current role is: FOLLOWER
14:14:18.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.910015 20443 sys_catalog.cc:565] T 00000000000000000000000000000000 P 1a132e8fe7dd4ee594ce6cbf8a76b157 [sys.catalog]: configured and running, proceeding with master startup.
14:14:18.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.913486 20362 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 05624e87b9a943dcb29bebb3750314e5: Neither blocks nor log segments found. Creating new log.
14:14:18.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.915901 20362 log.cc:826] T 00000000000000000000000000000000 P 05624e87b9a943dcb29bebb3750314e5: Log is configured to *not* fsync() on all Append() calls
14:14:18.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.920650 20362 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 05624e87b9a943dcb29bebb3750314e5: No bootstrap required, opened a new log
14:14:18.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:18.940994 20465 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 1a132e8fe7dd4ee594ce6cbf8a76b157: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:14:18.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:18.941449 20465 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:14:18.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.943475 20362 raft_consensus.cc:359] T 00000000000000000000000000000000 P 05624e87b9a943dcb29bebb3750314e5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "92e8830b9a8c416497ac98afc1c5468a" member_type: VOTER last_known_addr { host: "127.19.187.254" port: 42383 } } peers { permanent_uuid: "05624e87b9a943dcb29bebb3750314e5" member_type: VOTER last_known_addr { host: "127.19.187.253" port: 36413 } } peers { permanent_uuid: "1a132e8fe7dd4ee594ce6cbf8a76b157" member_type: VOTER last_known_addr { host: "127.19.187.252" port: 37535 } }
14:14:18.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.944059 20362 raft_consensus.cc:385] T 00000000000000000000000000000000 P 05624e87b9a943dcb29bebb3750314e5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:14:18.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.944334 20362 raft_consensus.cc:740] T 00000000000000000000000000000000 P 05624e87b9a943dcb29bebb3750314e5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 05624e87b9a943dcb29bebb3750314e5, State: Initialized, Role: FOLLOWER
14:14:18.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.945143 20362 consensus_queue.cc:260] T 00000000000000000000000000000000 P 05624e87b9a943dcb29bebb3750314e5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "92e8830b9a8c416497ac98afc1c5468a" member_type: VOTER last_known_addr { host: "127.19.187.254" port: 42383 } } peers { permanent_uuid: "05624e87b9a943dcb29bebb3750314e5" member_type: VOTER last_known_addr { host: "127.19.187.253" port: 36413 } } peers { permanent_uuid: "1a132e8fe7dd4ee594ce6cbf8a76b157" member_type: VOTER last_known_addr { host: "127.19.187.252" port: 37535 } }
14:14:18.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.952306 20467 sys_catalog.cc:455] T 00000000000000000000000000000000 P 05624e87b9a943dcb29bebb3750314e5 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "92e8830b9a8c416497ac98afc1c5468a" member_type: VOTER last_known_addr { host: "127.19.187.254" port: 42383 } } peers { permanent_uuid: "05624e87b9a943dcb29bebb3750314e5" member_type: VOTER last_known_addr { host: "127.19.187.253" port: 36413 } } peers { permanent_uuid: "1a132e8fe7dd4ee594ce6cbf8a76b157" member_type: VOTER last_known_addr { host: "127.19.187.252" port: 37535 } } }
14:14:18.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.953248 20467 sys_catalog.cc:458] T 00000000000000000000000000000000 P 05624e87b9a943dcb29bebb3750314e5 [sys.catalog]: This master's current role is: FOLLOWER
14:14:18.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.954213 20362 sys_catalog.cc:565] T 00000000000000000000000000000000 P 05624e87b9a943dcb29bebb3750314e5 [sys.catalog]: configured and running, proceeding with master startup.
14:14:18.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:18.958937 20316 tablet.cc:2378] T 00000000000000000000000000000000 P 05624e87b9a943dcb29bebb3750314e5: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:14:18.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.936002 20364 master_runner.cc:186] Error getting master registration for 127.19.187.254:42383: OK, instance_id { permanent_uuid: "92e8830b9a8c416497ac98afc1c5468a" instance_seqno: 1761401654750483 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:14:18.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.969851 20364 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:14:18.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:18.984297 20397 tablet.cc:2378] T 00000000000000000000000000000000 P 1a132e8fe7dd4ee594ce6cbf8a76b157: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:14:18.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:18.996526 20478 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 05624e87b9a943dcb29bebb3750314e5: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:14:18.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:18.996868 20478 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:14:19.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:18.982812 20289 master_runner.cc:186] Error getting master registration for 127.19.187.254:42383: OK, instance_id { permanent_uuid: "92e8830b9a8c416497ac98afc1c5468a" instance_seqno: 1761401654750483 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:14:19.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.015226 20289 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:14:19.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.064817 20453 raft_consensus.cc:493] T 00000000000000000000000000000000 P 1a132e8fe7dd4ee594ce6cbf8a76b157 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:14:19.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.065387 20453 raft_consensus.cc:515] T 00000000000000000000000000000000 P 1a132e8fe7dd4ee594ce6cbf8a76b157 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "92e8830b9a8c416497ac98afc1c5468a" member_type: VOTER last_known_addr { host: "127.19.187.254" port: 42383 } } peers { permanent_uuid: "05624e87b9a943dcb29bebb3750314e5" member_type: VOTER last_known_addr { host: "127.19.187.253" port: 36413 } } peers { permanent_uuid: "1a132e8fe7dd4ee594ce6cbf8a76b157" member_type: VOTER last_known_addr { host: "127.19.187.252" port: 37535 } }
14:14:19.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.068271 20453 leader_election.cc:290] T 00000000000000000000000000000000 P 1a132e8fe7dd4ee594ce6cbf8a76b157 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 92e8830b9a8c416497ac98afc1c5468a (127.19.187.254:42383), 05624e87b9a943dcb29bebb3750314e5 (127.19.187.253:36413)
14:14:19.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.069761 20337 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1a132e8fe7dd4ee594ce6cbf8a76b157" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "05624e87b9a943dcb29bebb3750314e5" is_pre_election: true
14:14:19.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.070627 20337 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 05624e87b9a943dcb29bebb3750314e5 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1a132e8fe7dd4ee594ce6cbf8a76b157 in term 0.
14:14:19.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.071970 20393 leader_election.cc:304] T 00000000000000000000000000000000 P 1a132e8fe7dd4ee594ce6cbf8a76b157 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 05624e87b9a943dcb29bebb3750314e5, 1a132e8fe7dd4ee594ce6cbf8a76b157; no voters:
14:14:19.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.072768 20453 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 1a132e8fe7dd4ee594ce6cbf8a76b157 [term 0 FOLLOWER]: Leader pre-election won for term 1
14:14:19.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.073101 20453 raft_consensus.cc:493] T 00000000000000000000000000000000 P 1a132e8fe7dd4ee594ce6cbf8a76b157 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:14:19.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.073372 20453 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 1a132e8fe7dd4ee594ce6cbf8a76b157 [term 0 FOLLOWER]: Advancing to term 1
14:14:19.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.081842 20453 raft_consensus.cc:515] T 00000000000000000000000000000000 P 1a132e8fe7dd4ee594ce6cbf8a76b157 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "92e8830b9a8c416497ac98afc1c5468a" member_type: VOTER last_known_addr { host: "127.19.187.254" port: 42383 } } peers { permanent_uuid: "05624e87b9a943dcb29bebb3750314e5" member_type: VOTER last_known_addr { host: "127.19.187.253" port: 36413 } } peers { permanent_uuid: "1a132e8fe7dd4ee594ce6cbf8a76b157" member_type: VOTER last_known_addr { host: "127.19.187.252" port: 37535 } }
14:14:19.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.090397 20453 leader_election.cc:290] T 00000000000000000000000000000000 P 1a132e8fe7dd4ee594ce6cbf8a76b157 [CANDIDATE]: Term 1 election: Requested vote from peers 92e8830b9a8c416497ac98afc1c5468a (127.19.187.254:42383), 05624e87b9a943dcb29bebb3750314e5 (127.19.187.253:36413)
14:14:19.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.091544 20337 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1a132e8fe7dd4ee594ce6cbf8a76b157" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "05624e87b9a943dcb29bebb3750314e5"
14:14:19.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.092087 20337 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 05624e87b9a943dcb29bebb3750314e5 [term 0 FOLLOWER]: Advancing to term 1
14:14:19.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.102447 20337 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 05624e87b9a943dcb29bebb3750314e5 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1a132e8fe7dd4ee594ce6cbf8a76b157 in term 1.
14:14:19.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.103967 20393 leader_election.cc:304] T 00000000000000000000000000000000 P 1a132e8fe7dd4ee594ce6cbf8a76b157 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 05624e87b9a943dcb29bebb3750314e5, 1a132e8fe7dd4ee594ce6cbf8a76b157; no voters:
14:14:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.104908 20453 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 1a132e8fe7dd4ee594ce6cbf8a76b157 [term 1 FOLLOWER]: Leader election won for term 1
14:14:19.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.111933 20453 raft_consensus.cc:697] T 00000000000000000000000000000000 P 1a132e8fe7dd4ee594ce6cbf8a76b157 [term 1 LEADER]: Becoming Leader. State: Replica: 1a132e8fe7dd4ee594ce6cbf8a76b157, State: Running, Role: LEADER
14:14:19.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.113540 20453 consensus_queue.cc:237] T 00000000000000000000000000000000 P 1a132e8fe7dd4ee594ce6cbf8a76b157 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "92e8830b9a8c416497ac98afc1c5468a" member_type: VOTER last_known_addr { host: "127.19.187.254" port: 42383 } } peers { permanent_uuid: "05624e87b9a943dcb29bebb3750314e5" member_type: VOTER last_known_addr { host: "127.19.187.253" port: 36413 } } peers { permanent_uuid: "1a132e8fe7dd4ee594ce6cbf8a76b157" member_type: VOTER last_known_addr { host: "127.19.187.252" port: 37535 } }
14:14:19.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.127302 20480 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1a132e8fe7dd4ee594ce6cbf8a76b157 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1a132e8fe7dd4ee594ce6cbf8a76b157. Latest consensus state: current_term: 1 leader_uuid: "1a132e8fe7dd4ee594ce6cbf8a76b157" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "92e8830b9a8c416497ac98afc1c5468a" member_type: VOTER last_known_addr { host: "127.19.187.254" port: 42383 } } peers { permanent_uuid: "05624e87b9a943dcb29bebb3750314e5" member_type: VOTER last_known_addr { host: "127.19.187.253" port: 36413 } } peers { permanent_uuid: "1a132e8fe7dd4ee594ce6cbf8a76b157" member_type: VOTER last_known_addr { host: "127.19.187.252" port: 37535 } } }
14:14:19.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.129822 20480 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1a132e8fe7dd4ee594ce6cbf8a76b157 [sys.catalog]: This master's current role is: LEADER
14:14:19.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.133606 20482 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
14:14:19.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.145220 20482 catalog_manager.cc:1494] Initializing Kudu cluster ID...
14:14:19.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.169683 20337 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 05624e87b9a943dcb29bebb3750314e5 [term 1 FOLLOWER]: Refusing update from remote peer 1a132e8fe7dd4ee594ce6cbf8a76b157: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:14:19.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.171264 20480 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 1a132e8fe7dd4ee594ce6cbf8a76b157 [LEADER]: Connected to new peer: Peer: permanent_uuid: "05624e87b9a943dcb29bebb3750314e5" member_type: VOTER last_known_addr { host: "127.19.187.253" port: 36413 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
14:14:19.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.202778 20467 sys_catalog.cc:455] T 00000000000000000000000000000000 P 05624e87b9a943dcb29bebb3750314e5 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1a132e8fe7dd4ee594ce6cbf8a76b157. Latest consensus state: current_term: 1 leader_uuid: "1a132e8fe7dd4ee594ce6cbf8a76b157" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "92e8830b9a8c416497ac98afc1c5468a" member_type: VOTER last_known_addr { host: "127.19.187.254" port: 42383 } } peers { permanent_uuid: "05624e87b9a943dcb29bebb3750314e5" member_type: VOTER last_known_addr { host: "127.19.187.253" port: 36413 } } peers { permanent_uuid: "1a132e8fe7dd4ee594ce6cbf8a76b157" member_type: VOTER last_known_addr { host: "127.19.187.252" port: 37535 } } }
14:14:19.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.207175 20467 sys_catalog.cc:458] T 00000000000000000000000000000000 P 05624e87b9a943dcb29bebb3750314e5 [sys.catalog]: This master's current role is: FOLLOWER
14:14:19.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.216383 20453 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1a132e8fe7dd4ee594ce6cbf8a76b157 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "1a132e8fe7dd4ee594ce6cbf8a76b157" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "92e8830b9a8c416497ac98afc1c5468a" member_type: VOTER last_known_addr { host: "127.19.187.254" port: 42383 } } peers { permanent_uuid: "05624e87b9a943dcb29bebb3750314e5" member_type: VOTER last_known_addr { host: "127.19.187.253" port: 36413 } } peers { permanent_uuid: "1a132e8fe7dd4ee594ce6cbf8a76b157" member_type: VOTER last_known_addr { host: "127.19.187.252" port: 37535 } } }
14:14:19.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.217108 20453 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1a132e8fe7dd4ee594ce6cbf8a76b157 [sys.catalog]: This master's current role is: LEADER
14:14:19.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.221015 20484 mvcc.cc:204] Tried to move back new op lower bound from 7214701195943661568 to 7214701195746918400. Current Snapshot: MvccSnapshot[applied={T|T < 7214701195943661568}]
14:14:19.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.224931 20482 catalog_manager.cc:1357] Generated new cluster ID: ea31ba4bccc942eca0b150703f6a9404
14:14:19.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.225210 20482 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
14:14:19.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.229481 20467 sys_catalog.cc:455] T 00000000000000000000000000000000 P 05624e87b9a943dcb29bebb3750314e5 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "1a132e8fe7dd4ee594ce6cbf8a76b157" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "92e8830b9a8c416497ac98afc1c5468a" member_type: VOTER last_known_addr { host: "127.19.187.254" port: 42383 } } peers { permanent_uuid: "05624e87b9a943dcb29bebb3750314e5" member_type: VOTER last_known_addr { host: "127.19.187.253" port: 36413 } } peers { permanent_uuid: "1a132e8fe7dd4ee594ce6cbf8a76b157" member_type: VOTER last_known_addr { host: "127.19.187.252" port: 37535 } } }
14:14:19.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.230171 20467 sys_catalog.cc:458] T 00000000000000000000000000000000 P 05624e87b9a943dcb29bebb3750314e5 [sys.catalog]: This master's current role is: FOLLOWER
14:14:19.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.314960 20482 catalog_manager.cc:1380] Generated new certificate authority record
14:14:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.318120 20482 catalog_manager.cc:1514] Loading token signing keys...
14:14:19.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.345340 20482 catalog_manager.cc:6022] T 00000000000000000000000000000000 P 1a132e8fe7dd4ee594ce6cbf8a76b157: Generated new TSK 0
14:14:19.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.346906 20482 catalog_manager.cc:1524] Initializing in-progress tserver states...
14:14:19.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.381986 20207 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu
14:14:19.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu
14:14:19.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-0/wal
14:14:19.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-0/data
14:14:19.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:14:19.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:14:19.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:14:19.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:19.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:19.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:19.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-0/logs
14:14:19.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-0/data/info.pb
14:14:19.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:19.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:14:19.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:14:19.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:14:19.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:14:19.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:19.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:19.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:19.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:14:19.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:14:19.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.187.193:0
14:14:19.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.187.193
14:14:19.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.187.193
14:14:19.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:14:19.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.187.254:42383,127.19.187.253:36413,127.19.187.252:37535
14:14:19.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.187.212:40939
14:14:19.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:19.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:19.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:14:19.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.512799 20287 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.187.252" port: 37535 } attempt: 7
14:14:19.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.583005 20287 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 92e8830b9a8c416497ac98afc1c5468a: Bootstrap starting.
14:14:19.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.604705 20287 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 92e8830b9a8c416497ac98afc1c5468a: Neither blocks nor log segments found. Creating new log.
14:14:19.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.610798 20287 log.cc:826] T 00000000000000000000000000000000 P 92e8830b9a8c416497ac98afc1c5468a: Log is configured to *not* fsync() on all Append() calls
14:14:19.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.630714 20287 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 92e8830b9a8c416497ac98afc1c5468a: No bootstrap required, opened a new log
14:14:19.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.662696 20287 raft_consensus.cc:359] T 00000000000000000000000000000000 P 92e8830b9a8c416497ac98afc1c5468a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "92e8830b9a8c416497ac98afc1c5468a" member_type: VOTER last_known_addr { host: "127.19.187.254" port: 42383 } } peers { permanent_uuid: "05624e87b9a943dcb29bebb3750314e5" member_type: VOTER last_known_addr { host: "127.19.187.253" port: 36413 } } peers { permanent_uuid: "1a132e8fe7dd4ee594ce6cbf8a76b157" member_type: VOTER last_known_addr { host: "127.19.187.252" port: 37535 } }
14:14:19.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.664074 20287 raft_consensus.cc:385] T 00000000000000000000000000000000 P 92e8830b9a8c416497ac98afc1c5468a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:14:19.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.664393 20287 raft_consensus.cc:740] T 00000000000000000000000000000000 P 92e8830b9a8c416497ac98afc1c5468a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 92e8830b9a8c416497ac98afc1c5468a, State: Initialized, Role: FOLLOWER
14:14:19.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.665680 20287 consensus_queue.cc:260] T 00000000000000000000000000000000 P 92e8830b9a8c416497ac98afc1c5468a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "92e8830b9a8c416497ac98afc1c5468a" member_type: VOTER last_known_addr { host: "127.19.187.254" port: 42383 } } peers { permanent_uuid: "05624e87b9a943dcb29bebb3750314e5" member_type: VOTER last_known_addr { host: "127.19.187.253" port: 36413 } } peers { permanent_uuid: "1a132e8fe7dd4ee594ce6cbf8a76b157" member_type: VOTER last_known_addr { host: "127.19.187.252" port: 37535 } }
14:14:19.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.673923 20492 sys_catalog.cc:455] T 00000000000000000000000000000000 P 92e8830b9a8c416497ac98afc1c5468a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "92e8830b9a8c416497ac98afc1c5468a" member_type: VOTER last_known_addr { host: "127.19.187.254" port: 42383 } } peers { permanent_uuid: "05624e87b9a943dcb29bebb3750314e5" member_type: VOTER last_known_addr { host: "127.19.187.253" port: 36413 } } peers { permanent_uuid: "1a132e8fe7dd4ee594ce6cbf8a76b157" member_type: VOTER last_known_addr { host: "127.19.187.252" port: 37535 } } }
14:14:19.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.674633 20492 sys_catalog.cc:458] T 00000000000000000000000000000000 P 92e8830b9a8c416497ac98afc1c5468a [sys.catalog]: This master's current role is: FOLLOWER
14:14:19.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.677780 20287 sys_catalog.cc:565] T 00000000000000000000000000000000 P 92e8830b9a8c416497ac98afc1c5468a [sys.catalog]: configured and running, proceeding with master startup.
14:14:19.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.678005 20261 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1a132e8fe7dd4ee594ce6cbf8a76b157" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "92e8830b9a8c416497ac98afc1c5468a"
14:14:19.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.678810 20261 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 92e8830b9a8c416497ac98afc1c5468a [term 0 FOLLOWER]: Advancing to term 1
14:14:19.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.683872 20262 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1a132e8fe7dd4ee594ce6cbf8a76b157" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "92e8830b9a8c416497ac98afc1c5468a" is_pre_election: true
14:14:19.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.687512 20261 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 92e8830b9a8c416497ac98afc1c5468a [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1a132e8fe7dd4ee594ce6cbf8a76b157 in term 1.
14:14:19.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.688743 20260 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 92e8830b9a8c416497ac98afc1c5468a [term 1 FOLLOWER]: Refusing update from remote peer 1a132e8fe7dd4ee594ce6cbf8a76b157: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:14:19.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.694659 20480 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 1a132e8fe7dd4ee594ce6cbf8a76b157 [LEADER]: Connected to new peer: Peer: permanent_uuid: "92e8830b9a8c416497ac98afc1c5468a" member_type: VOTER last_known_addr { host: "127.19.187.254" port: 42383 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:14:19.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:19.711705 20241 tablet.cc:2378] T 00000000000000000000000000000000 P 92e8830b9a8c416497ac98afc1c5468a: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:14:19.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.786947 20492 sys_catalog.cc:455] T 00000000000000000000000000000000 P 92e8830b9a8c416497ac98afc1c5468a [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1a132e8fe7dd4ee594ce6cbf8a76b157. Latest consensus state: current_term: 1 leader_uuid: "1a132e8fe7dd4ee594ce6cbf8a76b157" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "92e8830b9a8c416497ac98afc1c5468a" member_type: VOTER last_known_addr { host: "127.19.187.254" port: 42383 } } peers { permanent_uuid: "05624e87b9a943dcb29bebb3750314e5" member_type: VOTER last_known_addr { host: "127.19.187.253" port: 36413 } } peers { permanent_uuid: "1a132e8fe7dd4ee594ce6cbf8a76b157" member_type: VOTER last_known_addr { host: "127.19.187.252" port: 37535 } } }
14:14:19.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.786921 20500 sys_catalog.cc:455] T 00000000000000000000000000000000 P 92e8830b9a8c416497ac98afc1c5468a [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "1a132e8fe7dd4ee594ce6cbf8a76b157" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "92e8830b9a8c416497ac98afc1c5468a" member_type: VOTER last_known_addr { host: "127.19.187.254" port: 42383 } } peers { permanent_uuid: "05624e87b9a943dcb29bebb3750314e5" member_type: VOTER last_known_addr { host: "127.19.187.253" port: 36413 } } peers { permanent_uuid: "1a132e8fe7dd4ee594ce6cbf8a76b157" member_type: VOTER last_known_addr { host: "127.19.187.252" port: 37535 } } }
14:14:19.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.787704 20492 sys_catalog.cc:458] T 00000000000000000000000000000000 P 92e8830b9a8c416497ac98afc1c5468a [sys.catalog]: This master's current role is: FOLLOWER
14:14:19.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.787716 20500 sys_catalog.cc:458] T 00000000000000000000000000000000 P 92e8830b9a8c416497ac98afc1c5468a [sys.catalog]: This master's current role is: FOLLOWER
14:14:19.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:19.844007 20508 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
14:14:19.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:19.980523 20488 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:14:19.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:19.981115 20488 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:14:19.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:19.981827 20488 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:14:20.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:20.002435 20478 catalog_manager.cc:1269] Loaded cluster ID: ea31ba4bccc942eca0b150703f6a9404
14:14:20.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:20.003270 20478 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 05624e87b9a943dcb29bebb3750314e5: loading cluster ID for follower catalog manager: success
14:14:20.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:20.010252 20478 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 05624e87b9a943dcb29bebb3750314e5: acquiring CA information for follower catalog manager: success
14:14:20.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:20.015925 20478 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 05624e87b9a943dcb29bebb3750314e5: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:14:20.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:20.038321 20488 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:14:20.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:20.039724 20488 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.19.187.193
14:14:20.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:20.101460 20488 tablet_server_runner.cc:78] Tablet server non-default flags:
14:14:20.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:20.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.187.212:40939
14:14:20.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:20.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:14:20.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-0/data
14:14:20.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-0/wal
14:14:20.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:20.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:20.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.187.193:0
14:14:20.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:14:20.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:20.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:20.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-0/data/info.pb
14:14:20.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.187.193
14:14:20.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:14:20.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.187.254:42383,127.19.187.253:36413,127.19.187.252:37535
14:14:20.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:14:20.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:20.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:14:20.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:14:20.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:20.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.187.193
14:14:20.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-0/logs
14:14:20.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:20.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:14:20.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:14:20.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:14:20.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:14:20.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d0cb013a5aad754f2ed06c25f5ec3d8a1873622
14:14:20.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:14:20.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Oct 2025 13:43:23 UTC on 5fd53c4cbb9d
14:14:20.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8713
14:14:20.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:14:20.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:20.103119 20488 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:14:20.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:20.105121 20488 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:14:20.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:20.120038 20514 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:20.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:20.273928 20521 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1a132e8fe7dd4ee594ce6cbf8a76b157 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "1a132e8fe7dd4ee594ce6cbf8a76b157" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "92e8830b9a8c416497ac98afc1c5468a" member_type: VOTER last_known_addr { host: "127.19.187.254" port: 42383 } } peers { permanent_uuid: "05624e87b9a943dcb29bebb3750314e5" member_type: VOTER last_known_addr { host: "127.19.187.253" port: 36413 } } peers { permanent_uuid: "1a132e8fe7dd4ee594ce6cbf8a76b157" member_type: VOTER last_known_addr { host: "127.19.187.252" port: 37535 } } }
14:14:20.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:20.280193 20521 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1a132e8fe7dd4ee594ce6cbf8a76b157 [sys.catalog]: This master's current role is: LEADER
14:14:20.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:20.867239 20508 catalog_manager.cc:1269] Loaded cluster ID: ea31ba4bccc942eca0b150703f6a9404
14:14:20.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:20.867523 20508 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 92e8830b9a8c416497ac98afc1c5468a: loading cluster ID for follower catalog manager: success
14:14:20.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:20.901832 20508 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 92e8830b9a8c416497ac98afc1c5468a: acquiring CA information for follower catalog manager: success
14:14:20.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:20.924863 20508 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 92e8830b9a8c416497ac98afc1c5468a: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:14:22.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:21.524292 20513 debug-util.cc:398] Leaking SignalData structure 0x7b0800037500 after lost signal to thread 20488
14:14:22.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:22.188922 20488 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.066s user 0.465s sys 0.838s
14:14:22.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:22.189270 20488 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.067s user 0.465s sys 0.838s
14:14:22.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:20.123040 20515 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:22.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:22.191857 20517 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:22.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:22.194833 20516 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2067 milliseconds
14:14:22.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:22.195290 20488 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:14:22.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:22.196712 20488 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:14:22.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:22.212097 20488 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:14:22.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:22.213589 20488 hybrid_clock.cc:648] HybridClock initialized: now 1761401662213516 us; error 80 us; skew 500 ppm
14:14:22.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:22.214681 20488 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:14:22.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:22.252140 20488 webserver.cc:492] Webserver started at http://127.19.187.193:37261/ using document root <none> and password file <none>
14:14:22.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:22.253572 20488 fs_manager.cc:362] Metadata directory not provided
14:14:22.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:22.253841 20488 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:14:22.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:22.254369 20488 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:14:22.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:22.260428 20488 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-0/data/instance:
14:14:22.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f24d1f651a9846c69daeb73f18b567d2"
14:14:22.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 14:14:22 on dist-test-slave-bg5s"
14:14:22.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:22.261822 20488 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-0/wal/instance:
14:14:22.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f24d1f651a9846c69daeb73f18b567d2"
14:14:22.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 14:14:22 on dist-test-slave-bg5s"
14:14:22.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:22.281235 20488 fs_manager.cc:696] Time spent creating directory manager: real 0.019s user 0.008s sys 0.012s
14:14:22.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:22.293735 20528 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:14:22.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:22.294869 20488 fs_manager.cc:730] Time spent opening block manager: real 0.009s user 0.001s sys 0.009s
14:14:22.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:22.295250 20488 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-0/data,/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-0/wal
14:14:22.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f24d1f651a9846c69daeb73f18b567d2"
14:14:22.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 14:14:22 on dist-test-slave-bg5s"
14:14:22.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:22.295629 20488 fs_report.cc:389] FS layout report
14:14:22.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:14:22.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-0/wal
14:14:22.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-0/wal
14:14:22.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-0/data/data
14:14:22.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:14:22.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:14:22.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:14:22.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:14:22.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:14:22.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:14:22.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:14:22.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:14:22.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:14:22.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:14:22.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:14:22.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:22.389441 20488 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:14:22.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:22.391115 20488 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:14:22.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:22.391623 20488 kserver.cc:163] Server-wide thread pool size limit: 3276
14:14:22.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:22.400473 20488 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:14:22.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:22.411837 20488 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:14:22.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:22.412109 20488 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:14:22.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:22.412415 20488 ts_tablet_manager.cc:616] Registered 0 tablets
14:14:22.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:22.412614 20488 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:14:22.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:22.705439 20488 rpc_server.cc:307] RPC server started. Bound to: 127.19.187.193:34089
14:14:22.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:22.705565 20640 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.187.193:34089 every 8 connection(s)
14:14:22.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:22.708869 20488 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-0/data/info.pb
14:14:22.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:22.719739 20207 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu as pid 20488
14:14:22.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:22.720242 20207 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-0/wal/instance
14:14:22.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:22.735682 20207 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu
14:14:22.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu
14:14:22.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-1/wal
14:14:22.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-1/data
14:14:22.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:14:22.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:14:22.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:14:22.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:22.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:22.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:22.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-1/logs
14:14:22.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-1/data/info.pb
14:14:22.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:22.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:14:22.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:14:22.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:14:22.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:14:22.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:22.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:22.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:22.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:14:22.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:14:22.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.187.194:0
14:14:22.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.187.194
14:14:22.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.187.194
14:14:22.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:14:22.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.187.254:42383,127.19.187.253:36413,127.19.187.252:37535
14:14:22.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.187.212:40939
14:14:22.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:22.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:22.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:14:22.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:22.760039 20641 heartbeater.cc:344] Connected to a master server at 127.19.187.252:37535
14:14:22.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:22.760526 20641 heartbeater.cc:461] Registering TS with master...
14:14:22.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:22.761690 20641 heartbeater.cc:507] Master 127.19.187.252:37535 requested a full tablet report, sending...
14:14:22.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:22.764730 20408 ts_manager.cc:194] Registered new tserver with Master: f24d1f651a9846c69daeb73f18b567d2 (127.19.187.193:34089)
14:14:22.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:22.767251 20408 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.19.187.193:58523
14:14:22.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:22.767689 20642 heartbeater.cc:344] Connected to a master server at 127.19.187.253:36413
14:14:22.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:22.768031 20642 heartbeater.cc:461] Registering TS with master...
14:14:22.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:22.768947 20642 heartbeater.cc:507] Master 127.19.187.253:36413 requested a full tablet report, sending...
14:14:22.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:22.772425 20327 ts_manager.cc:194] Registered new tserver with Master: f24d1f651a9846c69daeb73f18b567d2 (127.19.187.193:34089)
14:14:22.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:22.789511 20645 heartbeater.cc:344] Connected to a master server at 127.19.187.254:42383
14:14:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:22.789834 20645 heartbeater.cc:461] Registering TS with master...
14:14:22.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:22.790408 20645 heartbeater.cc:507] Master 127.19.187.254:42383 requested a full tablet report, sending...
14:14:22.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:22.793568 20252 ts_manager.cc:194] Registered new tserver with Master: f24d1f651a9846c69daeb73f18b567d2 (127.19.187.193:34089)
14:14:23.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:23.089522 20648 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:14:23.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:23.089962 20648 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:14:23.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:23.090425 20648 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:14:23.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:23.120648 20648 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:14:23.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:23.121518 20648 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.19.187.194
14:14:23.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:23.155308 20648 tablet_server_runner.cc:78] Tablet server non-default flags:
14:14:23.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:23.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.187.212:40939
14:14:23.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:23.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:14:23.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-1/data
14:14:23.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-1/wal
14:14:23.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:23.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:23.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.187.194:0
14:14:23.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:14:23.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:23.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:23.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-1/data/info.pb
14:14:23.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.187.194
14:14:23.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:14:23.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.187.254:42383,127.19.187.253:36413,127.19.187.252:37535
14:14:23.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:14:23.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:23.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:14:23.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:14:23.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:23.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.187.194
14:14:23.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-1/logs
14:14:23.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:23.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:14:23.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:14:23.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:14:23.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:14:23.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d0cb013a5aad754f2ed06c25f5ec3d8a1873622
14:14:23.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:14:23.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Oct 2025 13:43:23 UTC on 5fd53c4cbb9d
14:14:23.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8713
14:14:23.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:14:23.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:23.156718 20648 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:14:23.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:23.158294 20648 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:14:23.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:23.172201 20657 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:23.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:23.771554 20641 heartbeater.cc:499] Master 127.19.187.252:37535 was elected leader, sending a full tablet report...
14:14:24.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:23.173763 20658 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:24.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:24.572563 20656 debug-util.cc:398] Leaking SignalData structure 0x7b0800037500 after lost signal to thread 20648
14:14:24.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:24.981877 20648 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.810s user 0.743s sys 1.023s
14:14:24.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:24.982200 20648 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.811s user 0.743s sys 1.023s
14:14:24.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:24.985312 20659 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1809 milliseconds
14:14:24.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:24.986568 20660 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:24.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:24.986631 20648 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:14:24.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:24.987684 20648 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:14:24.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:24.990022 20648 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:14:24.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:24.991366 20648 hybrid_clock.cc:648] HybridClock initialized: now 1761401664991330 us; error 41 us; skew 500 ppm
14:14:24.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:24.992386 20648 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:14:24.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:24.998310 20648 webserver.cc:492] Webserver started at http://127.19.187.194:44207/ using document root <none> and password file <none>
14:14:24.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:24.999405 20648 fs_manager.cc:362] Metadata directory not provided
14:14:24.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:24.999644 20648 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:14:25.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:25.000078 20648 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:14:25.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:25.004174 20648 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-1/data/instance:
14:14:25.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "849f59542f104c1e8f62021c38ade3eb"
14:14:25.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 14:14:24 on dist-test-slave-bg5s"
14:14:25.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:25.005271 20648 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-1/wal/instance:
14:14:25.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "849f59542f104c1e8f62021c38ade3eb"
14:14:25.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 14:14:24 on dist-test-slave-bg5s"
14:14:25.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:25.012270 20648 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.006s sys 0.002s
14:14:25.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:25.017539 20668 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:14:25.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:25.018518 20648 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.003s
14:14:25.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:25.018850 20648 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-1/data,/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-1/wal
14:14:25.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "849f59542f104c1e8f62021c38ade3eb"
14:14:25.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 14:14:24 on dist-test-slave-bg5s"
14:14:25.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:25.019181 20648 fs_report.cc:389] FS layout report
14:14:25.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:14:25.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-1/wal
14:14:25.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-1/wal
14:14:25.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-1/data/data
14:14:25.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:14:25.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:14:25.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:14:25.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:14:25.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:14:25.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:14:25.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:14:25.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:14:25.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:14:25.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:14:25.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:14:25.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:25.099906 20648 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:14:25.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:25.101416 20648 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:14:25.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:25.101868 20648 kserver.cc:163] Server-wide thread pool size limit: 3276
14:14:25.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:25.104552 20648 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:14:25.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:25.108497 20648 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:14:25.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:25.108700 20648 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:14:25.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:25.109009 20648 ts_tablet_manager.cc:616] Registered 0 tablets
14:14:25.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:25.109171 20648 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:14:25.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:25.243613 20648 rpc_server.cc:307] RPC server started. Bound to: 127.19.187.194:40269
14:14:25.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:25.243714 20781 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.187.194:40269 every 8 connection(s)
14:14:25.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:25.245950 20648 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-1/data/info.pb
14:14:25.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:25.253315 20207 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu as pid 20648
14:14:25.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:25.254086 20207 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-1/wal/instance
14:14:25.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:25.269698 20207 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu
14:14:25.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu
14:14:25.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-2/wal
14:14:25.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-2/data
14:14:25.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:14:25.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:14:25.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:14:25.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:25.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:25.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:25.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-2/logs
14:14:25.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-2/data/info.pb
14:14:25.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:25.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:14:25.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:14:25.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:14:25.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:14:25.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:25.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:25.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:25.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:14:25.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:14:25.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.187.195:0
14:14:25.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.187.195
14:14:25.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.187.195
14:14:25.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:14:25.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.187.254:42383,127.19.187.253:36413,127.19.187.252:37535
14:14:25.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.187.212:40939
14:14:25.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:25.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:25.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:14:25.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:25.290206 20782 heartbeater.cc:344] Connected to a master server at 127.19.187.252:37535
14:14:25.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:25.290791 20782 heartbeater.cc:461] Registering TS with master...
14:14:25.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:25.292280 20782 heartbeater.cc:507] Master 127.19.187.252:37535 requested a full tablet report, sending...
14:14:25.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:25.296056 20408 ts_manager.cc:194] Registered new tserver with Master: 849f59542f104c1e8f62021c38ade3eb (127.19.187.194:40269)
14:14:25.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:25.297966 20408 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.19.187.194:46573
14:14:25.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:25.303731 20783 heartbeater.cc:344] Connected to a master server at 127.19.187.253:36413
14:14:25.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:25.304082 20783 heartbeater.cc:461] Registering TS with master...
14:14:25.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:25.304760 20783 heartbeater.cc:507] Master 127.19.187.253:36413 requested a full tablet report, sending...
14:14:25.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:25.307969 20327 ts_manager.cc:194] Registered new tserver with Master: 849f59542f104c1e8f62021c38ade3eb (127.19.187.194:40269)
14:14:25.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:25.311794 20784 heartbeater.cc:344] Connected to a master server at 127.19.187.254:42383
14:14:25.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:25.312193 20784 heartbeater.cc:461] Registering TS with master...
14:14:25.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:25.312924 20784 heartbeater.cc:507] Master 127.19.187.254:42383 requested a full tablet report, sending...
14:14:25.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:25.315079 20252 ts_manager.cc:194] Registered new tserver with Master: 849f59542f104c1e8f62021c38ade3eb (127.19.187.194:40269)
14:14:25.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:25.592557 20792 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:14:25.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:25.593034 20792 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:14:25.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:25.593457 20792 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:14:25.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:25.625751 20792 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:14:25.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:25.626514 20792 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.19.187.195
14:14:25.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:25.662142 20792 tablet_server_runner.cc:78] Tablet server non-default flags:
14:14:25.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:25.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.187.212:40939
14:14:25.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:25.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:14:25.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-2/data
14:14:25.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-2/wal
14:14:25.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:25.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:25.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.187.195:0
14:14:25.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:14:25.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:25.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:25.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-2/data/info.pb
14:14:25.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.187.195
14:14:25.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:14:25.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.187.254:42383,127.19.187.253:36413,127.19.187.252:37535
14:14:25.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:14:25.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:25.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:14:25.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:14:25.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:25.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.187.195
14:14:25.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-2/logs
14:14:25.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:25.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:14:25.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:14:25.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:14:25.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:14:25.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d0cb013a5aad754f2ed06c25f5ec3d8a1873622
14:14:25.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:14:25.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Oct 2025 13:43:23 UTC on 5fd53c4cbb9d
14:14:25.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8713
14:14:25.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:14:25.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:25.663420 20792 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:14:25.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:25.665014 20792 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:14:25.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:25.676992 20798 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:26.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:26.303578 20782 heartbeater.cc:499] Master 127.19.187.252:37535 was elected leader, sending a full tablet report...
14:14:27.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:25.677410 20799 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:27.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:27.014837 20801 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:27.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:27.018054 20800 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1334 milliseconds
14:14:27.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:27.017998 20792 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.341s user 0.426s sys 0.909s
14:14:27.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:27.018373 20792 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.341s user 0.426s sys 0.909s
14:14:27.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.018719 20792 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:14:27.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.020318 20792 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:14:27.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.022872 20792 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:14:27.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.024400 20792 hybrid_clock.cc:648] HybridClock initialized: now 1761401667024340 us; error 50 us; skew 500 ppm
14:14:27.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.025595 20792 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:14:27.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.034346 20792 webserver.cc:492] Webserver started at http://127.19.187.195:40393/ using document root <none> and password file <none>
14:14:27.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.035851 20792 fs_manager.cc:362] Metadata directory not provided
14:14:27.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.036167 20792 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:14:27.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.036834 20792 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:14:27.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.044071 20792 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-2/data/instance:
14:14:27.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "de66b16de3b74990b9843e0db82c9de5"
14:14:27.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 14:14:27 on dist-test-slave-bg5s"
14:14:27.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.045744 20792 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-2/wal/instance:
14:14:27.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "de66b16de3b74990b9843e0db82c9de5"
14:14:27.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 14:14:27 on dist-test-slave-bg5s"
14:14:27.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.055552 20792 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.008s sys 0.003s
14:14:27.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.061897 20809 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:14:27.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.062963 20792 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
14:14:27.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.063246 20792 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-2/data,/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-2/wal
14:14:27.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "de66b16de3b74990b9843e0db82c9de5"
14:14:27.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 14:14:27 on dist-test-slave-bg5s"
14:14:27.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.063586 20792 fs_report.cc:389] FS layout report
14:14:27.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:14:27.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-2/wal
14:14:27.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-2/wal
14:14:27.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-2/data/data
14:14:27.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:14:27.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:14:27.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:14:27.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:14:27.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:14:27.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:14:27.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:14:27.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:14:27.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:14:27.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:14:27.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:14:27.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.138486 20792 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:14:27.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.140045 20792 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:14:27.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.140413 20792 kserver.cc:163] Server-wide thread pool size limit: 3276
14:14:27.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.142760 20792 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:14:27.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.146852 20792 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:14:27.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.147038 20792 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:14:27.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.147230 20792 ts_tablet_manager.cc:616] Registered 0 tablets
14:14:27.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.147405 20792 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:14:27.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.293773 20792 rpc_server.cc:307] RPC server started. Bound to: 127.19.187.195:44015
14:14:27.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.293879 20921 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.187.195:44015 every 8 connection(s)
14:14:27.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.296221 20792 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-2/data/info.pb
14:14:27.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.304288 20207 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu as pid 20792
14:14:27.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.304816 20207 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster9192661261571729956/ts-2/wal/instance
14:14:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.340503 20923 heartbeater.cc:344] Connected to a master server at 127.19.187.253:36413
14:14:27.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.341038 20923 heartbeater.cc:461] Registering TS with master...
14:14:27.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.342387 20923 heartbeater.cc:507] Master 127.19.187.253:36413 requested a full tablet report, sending...
14:14:27.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.345636 20327 ts_manager.cc:194] Registered new tserver with Master: de66b16de3b74990b9843e0db82c9de5 (127.19.187.195:44015)
14:14:27.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.349089 20922 heartbeater.cc:344] Connected to a master server at 127.19.187.252:37535
14:14:27.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.349438 20922 heartbeater.cc:461] Registering TS with master...
14:14:27.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.350308 20922 heartbeater.cc:507] Master 127.19.187.252:37535 requested a full tablet report, sending...
14:14:27.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.351047 20924 heartbeater.cc:344] Connected to a master server at 127.19.187.254:42383
14:14:27.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.351326 20924 heartbeater.cc:461] Registering TS with master...
14:14:27.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.352185 20924 heartbeater.cc:507] Master 127.19.187.254:42383 requested a full tablet report, sending...
14:14:27.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.352797 20407 ts_manager.cc:194] Registered new tserver with Master: de66b16de3b74990b9843e0db82c9de5 (127.19.187.195:44015)
14:14:27.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.354207 20252 ts_manager.cc:194] Registered new tserver with Master: de66b16de3b74990b9843e0db82c9de5 (127.19.187.195:44015)
14:14:27.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.354467 20407 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.19.187.195:39099
14:14:27.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.370645 20207 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
14:14:27.381 [DEBUG - main] (MiniKuduCluster.java:195) Response:
14:14:27.381 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
14:14:27.382 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.19.187.254"
port: 42383
}
bound_http_address {
host: "127.19.187.254"
port: 43349
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.19.187.253"
port: 36413
}
bound_http_address {
host: "127.19.187.253"
port: 35661
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.19.187.252"
port: 37535
}
bound_http_address {
host: "127.19.187.252"
port: 35107
}
}
}
14:14:27.383 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
14:14:27.384 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.19.187.193"
port: 34089
}
bound_http_address {
host: "127.19.187.193"
port: 37261
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.19.187.194"
port: 40269
}
bound_http_address {
host: "127.19.187.194"
port: 44207
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.19.187.195"
port: 44015
}
bound_http_address {
host: "127.19.187.195"
port: 40393
}
}
}
14:14:27.385 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
14:14:27.386 [DEBUG - main] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster1071084657143595704/krb5kdc/krb5cc
14:14:27.387 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
14:14:27.387 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskB9qii1/test-tmp
14:14:27.388 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
14:14:27.388 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskB9qii1/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
14:14:27.389 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
14:14:27.719 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:14:27.720 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
14:14:27.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.722842 20933 mini_oidc.cc:150] Starting JWKS server
14:14:27.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.725801 20933 webserver.cc:492] Webserver started at http://127.0.0.1:33059/ using document root <none> and password file <none>
14:14:27.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.726173 20933 mini_oidc.cc:186] Starting OIDC Discovery server
14:14:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:27.727749 20933 webserver.cc:492] Webserver started at http://127.0.0.1:45331/ using document root <none> and password file <none>
14:14:27.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
14:14:27.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/principal' for realm 'KRBTEST.COM',
14:14:27.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
14:14:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 25 14:14:27 dist-test-slave-bg5s krb5kdc[20944](info): setting up network...
14:14:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
14:14:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 25 14:14:27 dist-test-slave-bg5s krb5kdc[20944](info): set up 2 sockets
14:14:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 25 14:14:27 dist-test-slave-bg5s krb5kdc[20944](info): commencing operation
14:14:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
14:14:28.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:28.357275 20922 heartbeater.cc:499] Master 127.19.187.252:37535 was elected leader, sending a full tablet report...
14:14:29.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:29.716596 20933 mini_kdc.cc:121] Time spent starting KDC: real 1.951s user 0.000s sys 0.018s
14:14:29.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
14:14:29.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:14:29.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
14:14:29.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
14:14:29.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:14:29.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
14:14:29.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
14:14:29.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:14:29.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
14:14:29.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:14:29.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/test-user.keytab.
14:14:29.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/test-user.keytab.
14:14:29.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 25 14:14:29 dist-test-slave-bg5s krb5kdc[20944](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761401669, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:14:29.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
14:14:29.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-25T14:14:29Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
14:14:29.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-25T14:14:29Z Disabled control of system clock
14:14:29.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.20.113.126@KRBTEST.COM; defaulting to no policy
14:14:29.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:14:29.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.20.113.126@KRBTEST.COM" created.
14:14:29.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:14:29.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.20.113.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kudu.keytab.
14:14:29.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.20.113.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kudu.keytab.
14:14:29.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.20.113.126@KRBTEST.COM; defaulting to no policy
14:14:29.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:14:29.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.20.113.126@KRBTEST.COM" created.
14:14:29.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:14:29.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.20.113.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kudu.keytab.
14:14:29.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.20.113.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kudu.keytab.
14:14:29.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:29.938108 20933 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu
14:14:29.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu
14:14:29.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/wal
14:14:29.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/data
14:14:29.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:14:29.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:14:29.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:14:29.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:29.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:29.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:29.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/logs
14:14:29.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/data/info.pb
14:14:29.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:29.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:14:29.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:14:29.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:14:29.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:14:29.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:29.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:29.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:29.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:14:29.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:14:29.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:14:29.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:14:29.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.113.126:44003
14:14:29.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.113.126
14:14:29.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:14:29.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kudu.keytab
14:14:29.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.113.126
14:14:29.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:14:29.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:14:29.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:14:29.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:14:29.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.113.84:43553
14:14:29.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:29.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:29.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:14:29.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:14:29.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.113.126:44003,127.20.113.125:43287,127.20.113.124:34873
14:14:29.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:14:29.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33059/jwks.json
14:14:29.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:14:29.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:14:30.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:30.226563 20961 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:14:30.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:30.227200 20961 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:14:30.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:30.227947 20961 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:14:30.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:30.265655 20961 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:14:30.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:30.265918 20961 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:14:30.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:30.266330 20961 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:14:30.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:30.266633 20961 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:14:30.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:30.266947 20961 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:14:30.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:30.267210 20961 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:14:30.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:30.267452 20961 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33059/jwks.json
14:14:30.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:30.303544 20961 master_runner.cc:386] Master server non-default flags:
14:14:30.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:30.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.113.84:43553
14:14:30.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:30.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:14:30.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/data
14:14:30.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/wal
14:14:30.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:14:30.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:14:30.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.113.126:44003,127.20.113.125:43287,127.20.113.124:34873
14:14:30.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kudu.keytab
14:14:30.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.113.126
14:14:30.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:30.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:30.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:14:30.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.113.126:44003
14:14:30.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:14:30.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:14:30.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:14:30.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33059/jwks.json
14:14:30.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:14:30.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:14:30.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:14:30.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:30.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:30.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/data/info.pb
14:14:30.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.113.126
14:14:30.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:14:30.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:14:30.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:30.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:14:30.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:14:30.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:30.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/logs
14:14:30.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:30.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:14:30.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:14:30.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:14:30.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:14:30.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d0cb013a5aad754f2ed06c25f5ec3d8a1873622
14:14:30.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:14:30.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Oct 2025 13:43:23 UTC on 5fd53c4cbb9d
14:14:30.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8713
14:14:30.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:14:30.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:30.305074 20961 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:14:30.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:30.306233 20961 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:14:30.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:30.316135 20967 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:31.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:30.317765 20968 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:31.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:31.516454 20970 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:31.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:31.518555 20969 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1198 milliseconds
14:14:31.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:31.518685 20961 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:14:31.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:31.519989 20961 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:14:31.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:31.522843 20961 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:14:31.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:31.524366 20961 hybrid_clock.cc:648] HybridClock initialized: now 1761401671524317 us; error 50 us; skew 500 ppm
14:14:31.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:31.525203 20961 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:14:31.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 25 14:14:31 dist-test-slave-bg5s krb5kdc[20944](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761401671, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.113.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:14:31.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:31.531303 20961 init.cc:377] Logged in from keytab as kudu/127.20.113.126@KRBTEST.COM (short username kudu)
14:14:31.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:31.537739 20961 webserver.cc:492] Webserver started at http://127.20.113.126:45853/ using document root <none> and password file <none>
14:14:31.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:31.539055 20961 fs_manager.cc:362] Metadata directory not provided
14:14:31.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:31.539314 20961 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:14:31.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:31.539878 20961 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:14:31.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:31.544991 20961 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/data/instance:
14:14:31.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f7fa589d2c08468aa6cd09db9c033551"
14:14:31.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 14:14:31 on dist-test-slave-bg5s"
14:14:31.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:31.546144 20961 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/wal/instance:
14:14:31.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f7fa589d2c08468aa6cd09db9c033551"
14:14:31.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 14:14:31 on dist-test-slave-bg5s"
14:14:31.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:31.554196 20961 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
14:14:31.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:31.559975 20979 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:14:31.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:31.561074 20961 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
14:14:31.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:31.561381 20961 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/data,/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/wal
14:14:31.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f7fa589d2c08468aa6cd09db9c033551"
14:14:31.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 14:14:31 on dist-test-slave-bg5s"
14:14:31.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:31.561679 20961 fs_report.cc:389] FS layout report
14:14:31.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:14:31.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/wal
14:14:31.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/wal
14:14:31.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/data/data
14:14:31.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:14:31.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:14:31.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:14:31.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:14:31.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:14:31.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:14:31.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:14:31.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:14:31.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:14:31.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:14:31.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:14:31.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:31.611824 20961 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:14:31.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:31.617129 20961 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:14:31.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:31.617550 20961 kserver.cc:163] Server-wide thread pool size limit: 3276
14:14:31.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:31.685720 20961 rpc_server.cc:307] RPC server started. Bound to: 127.20.113.126:44003
14:14:31.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:31.685801 21030 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.113.126:44003 every 8 connection(s)
14:14:31.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:31.688328 20961 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/data/info.pb
14:14:31.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:31.693938 21031 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:14:31.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:31.698819 20933 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu as pid 20961
14:14:31.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:31.699304 20933 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/wal/instance
14:14:31.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:31.704329 21031 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } has no permanent_uuid. Determining permanent_uuid...
14:14:31.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.20.113.125@KRBTEST.COM; defaulting to no policy
14:14:31.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:14:31.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.20.113.125@KRBTEST.COM" created.
14:14:31.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 25 14:14:31 dist-test-slave-bg5s krb5kdc[20944](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761401671, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.113.126@KRBTEST.COM for kudu/127.20.113.126@KRBTEST.COM
14:14:31.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:14:31.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.20.113.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kudu.keytab.
14:14:31.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.20.113.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kudu.keytab.
14:14:31.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:31.753494 21031 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } has no permanent_uuid. Determining permanent_uuid...
14:14:31.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:31.756443 20983 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.113.125:43287: connect: Connection refused (error 111)
14:14:31.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.20.113.125@KRBTEST.COM; defaulting to no policy
14:14:31.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:14:31.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.20.113.125@KRBTEST.COM" created.
14:14:31.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:31.760594 21031 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.113.125:43287: Network error: Client connection negotiation failed: client connection to 127.20.113.125:43287: connect: Connection refused (error 111)
14:14:31.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:14:31.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.20.113.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kudu.keytab.
14:14:31.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.20.113.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kudu.keytab.
14:14:31.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:31.781508 20933 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu
14:14:31.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu
14:14:31.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/wal
14:14:31.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/data
14:14:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:14:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:14:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:14:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/logs
14:14:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/data/info.pb
14:14:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:14:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:14:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:14:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:14:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:14:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:14:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:14:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:14:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.113.125:43287
14:14:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.113.125
14:14:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:14:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kudu.keytab
14:14:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.113.125
14:14:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:14:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:14:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:14:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:14:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.113.84:43553
14:14:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:14:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:14:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.113.126:44003,127.20.113.125:43287,127.20.113.124:34873
14:14:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:14:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33059/jwks.json
14:14:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:14:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:14:31.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:31.810052 21031 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } attempt: 1
14:14:31.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:31.814155 21031 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.113.125:43287: Network error: Client connection negotiation failed: client connection to 127.20.113.125:43287: connect: Connection refused (error 111)
14:14:31.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:31.882632 21031 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } attempt: 2
14:14:31.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:31.886188 21031 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.113.125:43287: Network error: Client connection negotiation failed: client connection to 127.20.113.125:43287: connect: Connection refused (error 111)
14:14:31.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:31.977512 21031 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } attempt: 3
14:14:31.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:31.981449 21031 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.113.125:43287: Network error: Client connection negotiation failed: client connection to 127.20.113.125:43287: connect: Connection refused (error 111)
14:14:32.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:32.081625 21040 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:14:32.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:32.082135 21040 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:14:32.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:32.082720 21040 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:14:32.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:32.113246 21040 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:14:32.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:32.113456 21040 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:14:32.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:32.113701 21040 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:14:32.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:32.113888 21040 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:14:32.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:32.114076 21040 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:14:32.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:32.114251 21040 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:14:32.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:32.114380 21040 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33059/jwks.json
14:14:32.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:32.124958 21031 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } attempt: 4
14:14:32.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:32.128608 21031 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.113.125:43287: Network error: Client connection negotiation failed: client connection to 127.20.113.125:43287: connect: Connection refused (error 111)
14:14:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:32.152026 21040 master_runner.cc:386] Master server non-default flags:
14:14:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.113.84:43553
14:14:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:14:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/data
14:14:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/wal
14:14:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:14:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:14:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.113.126:44003,127.20.113.125:43287,127.20.113.124:34873
14:14:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kudu.keytab
14:14:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.113.125
14:14:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:14:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.113.125:43287
14:14:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:14:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:14:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:14:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33059/jwks.json
14:14:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:14:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:14:32.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:14:32.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:32.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:32.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/data/info.pb
14:14:32.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.113.125
14:14:32.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:14:32.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:14:32.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:32.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:14:32.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:14:32.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:32.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/logs
14:14:32.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:32.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:14:32.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:14:32.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:14:32.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:14:32.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d0cb013a5aad754f2ed06c25f5ec3d8a1873622
14:14:32.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:14:32.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Oct 2025 13:43:23 UTC on 5fd53c4cbb9d
14:14:32.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8713
14:14:32.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:14:32.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:32.153661 21040 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:14:32.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:32.155016 21040 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:14:32.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:32.165198 21046 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:32.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:32.428097 21031 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } attempt: 5
14:14:32.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:32.434455 21031 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.113.125:43287: Network error: Client connection negotiation failed: client connection to 127.20.113.125:43287: connect: Connection refused (error 111)
14:14:32.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:32.981963 21031 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } attempt: 6
14:14:32.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:32.992774 21031 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.113.125:43287: Network error: Client connection negotiation failed: client connection to 127.20.113.125:43287: connect: Connection refused (error 111)
14:14:33.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:32.167218 21047 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:33.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:32.177192 21049 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:33.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:33.448328 21048 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1278 milliseconds
14:14:33.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:33.448447 21040 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:14:33.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:33.449759 21040 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:14:33.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:33.452622 21040 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:14:33.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:33.454008 21040 hybrid_clock.cc:648] HybridClock initialized: now 1761401673453963 us; error 58 us; skew 500 ppm
14:14:33.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:33.454730 21040 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:14:33.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 25 14:14:33 dist-test-slave-bg5s krb5kdc[20944](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761401673, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.113.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:14:33.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:33.460692 21040 init.cc:377] Logged in from keytab as kudu/127.20.113.125@KRBTEST.COM (short username kudu)
14:14:33.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:33.465656 21040 webserver.cc:492] Webserver started at http://127.20.113.125:37077/ using document root <none> and password file <none>
14:14:33.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:33.466487 21040 fs_manager.cc:362] Metadata directory not provided
14:14:33.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:33.466652 21040 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:14:33.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:33.467018 21040 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:14:33.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:33.471091 21040 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/data/instance:
14:14:33.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "70c2c7e4f63b4d3f94198427af0a5a3e"
14:14:33.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 14:14:33 on dist-test-slave-bg5s"
14:14:33.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:33.472160 21040 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/wal/instance:
14:14:33.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "70c2c7e4f63b4d3f94198427af0a5a3e"
14:14:33.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 14:14:33 on dist-test-slave-bg5s"
14:14:33.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:33.479161 21040 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.005s sys 0.000s
14:14:33.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:33.484460 21062 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:14:33.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:33.485322 21040 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.002s
14:14:33.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:33.485646 21040 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/data,/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/wal
14:14:33.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "70c2c7e4f63b4d3f94198427af0a5a3e"
14:14:33.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 14:14:33 on dist-test-slave-bg5s"
14:14:33.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:33.485932 21040 fs_report.cc:389] FS layout report
14:14:33.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:14:33.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/wal
14:14:33.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/wal
14:14:33.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/data/data
14:14:33.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:14:33.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:14:33.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:14:33.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:14:33.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:14:33.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:14:33.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:14:33.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:14:33.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:14:33.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:14:33.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:14:33.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:33.546264 21040 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:14:33.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:33.554420 21040 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:14:33.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:33.554817 21040 kserver.cc:163] Server-wide thread pool size limit: 3276
14:14:33.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:33.627796 21040 rpc_server.cc:307] RPC server started. Bound to: 127.20.113.125:43287
14:14:33.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:33.627861 21113 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.113.125:43287 every 8 connection(s)
14:14:33.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:33.630496 21040 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/data/info.pb
14:14:33.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:33.636009 21114 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:14:33.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:33.636029 20933 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu as pid 21040
14:14:33.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:33.636365 20933 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/wal/instance
14:14:33.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:33.648947 21114 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } has no permanent_uuid. Determining permanent_uuid...
14:14:33.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.20.113.124@KRBTEST.COM; defaulting to no policy
14:14:33.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:14:33.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.20.113.124@KRBTEST.COM" created.
14:14:33.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 25 14:14:33 dist-test-slave-bg5s krb5kdc[20944](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761401673, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.113.125@KRBTEST.COM for kudu/127.20.113.126@KRBTEST.COM
14:14:33.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:14:33.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.20.113.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kudu.keytab.
14:14:33.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.20.113.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kudu.keytab.
14:14:33.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:33.682626 21114 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } has no permanent_uuid. Determining permanent_uuid...
14:14:33.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.20.113.124@KRBTEST.COM; defaulting to no policy
14:14:33.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:14:33.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.20.113.124@KRBTEST.COM" created.
14:14:33.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 25 14:14:33 dist-test-slave-bg5s krb5kdc[20944](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761401673, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.113.125@KRBTEST.COM for kudu/127.20.113.125@KRBTEST.COM
14:14:33.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:14:33.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.20.113.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kudu.keytab.
14:14:33.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.20.113.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kudu.keytab.
14:14:33.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:33.714669 21114 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } has no permanent_uuid. Determining permanent_uuid...
14:14:33.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:33.716990 20933 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu
14:14:33.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu
14:14:33.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/wal
14:14:33.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/data
14:14:33.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:14:33.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:14:33.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:14:33.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:33.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:33.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:33.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/logs
14:14:33.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/data/info.pb
14:14:33.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:33.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:14:33.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:14:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:14:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:14:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:14:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:14:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:14:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:14:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.113.124:34873
14:14:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.113.124
14:14:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:14:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kudu.keytab
14:14:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.113.124
14:14:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:14:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:14:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:14:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:14:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.113.84:43553
14:14:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:14:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:14:33.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.113.126:44003,127.20.113.125:43287,127.20.113.124:34873
14:14:33.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:14:33.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33059/jwks.json
14:14:33.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:14:33.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:14:33.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:33.717701 21064 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.113.124:34873: connect: Connection refused (error 111)
14:14:33.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:33.721881 21114 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.113.124:34873: Network error: Client connection negotiation failed: client connection to 127.20.113.124:34873: connect: Connection refused (error 111)
14:14:33.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:33.771345 21114 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } attempt: 1
14:14:33.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:33.775722 21114 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.113.124:34873: Network error: Client connection negotiation failed: client connection to 127.20.113.124:34873: connect: Connection refused (error 111)
14:14:33.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:33.844161 21114 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } attempt: 2
14:14:33.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:33.847950 21114 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.113.124:34873: Network error: Client connection negotiation failed: client connection to 127.20.113.124:34873: connect: Connection refused (error 111)
14:14:33.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:33.939350 21114 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } attempt: 3
14:14:33.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:33.943284 21114 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.113.124:34873: Network error: Client connection negotiation failed: client connection to 127.20.113.124:34873: connect: Connection refused (error 111)
14:14:34.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:34.023630 21124 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:14:34.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:34.024156 21124 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:14:34.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:34.024784 21124 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:14:34.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:34.053335 21031 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } attempt: 7
14:14:34.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:34.057965 21124 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:14:34.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:34.058188 21124 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:14:34.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:34.058590 21124 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:14:34.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:34.058785 21124 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:14:34.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:34.059100 21124 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:14:34.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:34.059418 21124 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:14:34.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:34.059677 21124 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33059/jwks.json
14:14:34.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 25 14:14:34 dist-test-slave-bg5s krb5kdc[20944](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761401671, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.113.126@KRBTEST.COM for kudu/127.20.113.125@KRBTEST.COM
14:14:34.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:34.074949 21031 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } has no permanent_uuid. Determining permanent_uuid...
14:14:34.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:34.081084 21031 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.113.124:34873: Network error: Client connection negotiation failed: client connection to 127.20.113.124:34873: connect: Connection refused (error 111)
14:14:34.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:34.086687 21114 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } attempt: 4
14:14:34.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:34.090039 21114 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.113.124:34873: Network error: Client connection negotiation failed: client connection to 127.20.113.124:34873: connect: Connection refused (error 111)
14:14:34.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:34.101395 21124 master_runner.cc:386] Master server non-default flags:
14:14:34.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:34.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.113.84:43553
14:14:34.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:34.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:14:34.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/data
14:14:34.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/wal
14:14:34.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:14:34.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:14:34.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.113.126:44003,127.20.113.125:43287,127.20.113.124:34873
14:14:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kudu.keytab
14:14:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.113.124
14:14:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:14:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.113.124:34873
14:14:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:14:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:14:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:14:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33059/jwks.json
14:14:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:14:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:14:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:14:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/data/info.pb
14:14:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.113.124
14:14:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:14:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:14:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:14:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:14:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/logs
14:14:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:14:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:14:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:14:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:14:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d0cb013a5aad754f2ed06c25f5ec3d8a1873622
14:14:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:14:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Oct 2025 13:43:23 UTC on 5fd53c4cbb9d
14:14:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8713
14:14:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:14:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:34.102905 21124 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:14:34.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:34.103916 21124 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:14:34.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:34.113658 21132 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:34.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:34.139506 21031 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } attempt: 1
14:14:34.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:34.146950 21031 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.113.124:34873: Network error: Client connection negotiation failed: client connection to 127.20.113.124:34873: connect: Connection refused (error 111)
14:14:34.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:34.228571 21031 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } attempt: 2
14:14:34.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:34.235993 21031 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.113.124:34873: Network error: Client connection negotiation failed: client connection to 127.20.113.124:34873: connect: Connection refused (error 111)
14:14:34.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:34.321569 21031 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } attempt: 3
14:14:34.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:34.328382 21031 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.113.124:34873: Network error: Client connection negotiation failed: client connection to 127.20.113.124:34873: connect: Connection refused (error 111)
14:14:34.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:34.389540 21114 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } attempt: 5
14:14:34.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:34.396564 21114 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.113.124:34873: Network error: Client connection negotiation failed: client connection to 127.20.113.124:34873: connect: Connection refused (error 111)
14:14:34.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:34.468922 21031 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } attempt: 4
14:14:34.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:34.483820 21031 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.113.124:34873: Network error: Client connection negotiation failed: client connection to 127.20.113.124:34873: connect: Connection refused (error 111)
14:14:34.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:34.767364 21031 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } attempt: 5
14:14:34.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:34.773341 21031 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.113.124:34873: Network error: Client connection negotiation failed: client connection to 127.20.113.124:34873: connect: Connection refused (error 111)
14:14:34.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:34.944105 21114 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } attempt: 6
14:14:34.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:34.975246 21114 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.113.124:34873: Network error: Client connection negotiation failed: client connection to 127.20.113.124:34873: connect: Connection refused (error 111)
14:14:35.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:35.325788 21031 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } attempt: 6
14:14:35.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:35.364562 21031 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.113.124:34873: Network error: Client connection negotiation failed: client connection to 127.20.113.124:34873: connect: Connection refused (error 111)
14:14:35.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:35.518096 21131 debug-util.cc:398] Leaking SignalData structure 0x7b0800027080 after lost signal to thread 21124
14:14:35.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:35.871683 21131 kernel_stack_watchdog.cc:198] Thread 21124 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
14:14:35.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:14:35.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:14:35.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:14:35.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:14:35.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:34.114449 21133 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:35.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:35.873356 21124 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.759s user 0.601s sys 1.107s
14:14:35.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:35.874472 21124 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.760s user 0.601s sys 1.108s
14:14:35.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:35.874804 21134 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1758 milliseconds
14:14:35.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:35.876024 21135 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:35.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:35.875975 21124 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:14:35.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:35.880052 21124 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:14:35.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:35.882390 21124 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:14:35.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:35.883791 21124 hybrid_clock.cc:648] HybridClock initialized: now 1761401675883747 us; error 57 us; skew 500 ppm
14:14:35.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:35.884631 21124 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:14:35.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 25 14:14:35 dist-test-slave-bg5s krb5kdc[20944](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761401675, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.113.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:14:35.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:35.890436 21124 init.cc:377] Logged in from keytab as kudu/127.20.113.124@KRBTEST.COM (short username kudu)
14:14:35.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:35.894670 21124 webserver.cc:492] Webserver started at http://127.20.113.124:40487/ using document root <none> and password file <none>
14:14:35.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:35.895524 21124 fs_manager.cc:362] Metadata directory not provided
14:14:35.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:35.895704 21124 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:14:35.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:35.896098 21124 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:14:35.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:35.900235 21124 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/data/instance:
14:14:35.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "661b46d073c04359b178727bdbe67c4d"
14:14:35.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 14:14:35 on dist-test-slave-bg5s"
14:14:35.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:35.901203 21124 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/wal/instance:
14:14:35.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "661b46d073c04359b178727bdbe67c4d"
14:14:35.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 14:14:35 on dist-test-slave-bg5s"
14:14:35.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:35.907845 21124 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.006s sys 0.001s
14:14:35.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:35.912923 21149 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:14:35.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:35.913900 21124 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.004s sys 0.001s
14:14:35.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:35.914161 21124 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/data,/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/wal
14:14:35.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "661b46d073c04359b178727bdbe67c4d"
14:14:35.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 14:14:35 on dist-test-slave-bg5s"
14:14:35.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:35.914430 21124 fs_report.cc:389] FS layout report
14:14:35.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:14:35.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/wal
14:14:35.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/wal
14:14:35.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/data/data
14:14:35.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:14:35.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:14:35.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:14:35.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:14:35.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:14:35.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:14:35.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:14:35.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:14:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:14:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:14:35.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:14:35.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:35.974081 21124 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:14:35.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:35.979542 21124 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:14:35.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:35.979928 21124 kserver.cc:163] Server-wide thread pool size limit: 3276
14:14:36.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.035845 21114 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } attempt: 7
14:14:36.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:36.043901 21114 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.113.124:34873: Network error: Client connection negotiation failed: client connection to 127.20.113.124:34873: connect: Connection refused (error 111)
14:14:36.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.048590 21124 rpc_server.cc:307] RPC server started. Bound to: 127.20.113.124:34873
14:14:36.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.048658 21202 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.113.124:34873 every 8 connection(s)
14:14:36.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.051283 21124 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/data/info.pb
14:14:36.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.057029 21203 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:14:36.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.060705 20933 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu as pid 21124
14:14:36.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.061158 20933 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/wal/instance
14:14:36.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.070389 21203 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } has no permanent_uuid. Determining permanent_uuid...
14:14:36.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 25 14:14:36 dist-test-slave-bg5s krb5kdc[20944](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761401675, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.113.124@KRBTEST.COM for kudu/127.20.113.126@KRBTEST.COM
14:14:36.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 25 14:14:36 dist-test-slave-bg5s krb5kdc[20944](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761401669, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.20.113.124@KRBTEST.COM
14:14:36.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.100445 21203 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } has no permanent_uuid. Determining permanent_uuid...
14:14:36.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 25 14:14:36 dist-test-slave-bg5s krb5kdc[20944](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761401675, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.113.124@KRBTEST.COM for kudu/127.20.113.125@KRBTEST.COM
14:14:36.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 25 14:14:36 dist-test-slave-bg5s krb5kdc[20944](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761401669, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.20.113.125@KRBTEST.COM
14:14:36.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.123823 21203 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } has no permanent_uuid. Determining permanent_uuid...
14:14:36.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 25 14:14:36 dist-test-slave-bg5s krb5kdc[20944](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761401675, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.113.124@KRBTEST.COM for kudu/127.20.113.124@KRBTEST.COM
14:14:36.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 25 14:14:36 dist-test-slave-bg5s krb5kdc[20944](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761401669, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.20.113.126@KRBTEST.COM
14:14:36.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.150903 21203 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d: Bootstrap starting.
14:14:36.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.157684 21203 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d: Neither blocks nor log segments found. Creating new log.
14:14:36.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.157809 20933 external_mini_cluster.cc:949] 0 TS(s) registered with all masters
14:14:36.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.159354 21203 log.cc:826] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d: Log is configured to *not* fsync() on all Append() calls
14:14:36.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.163749 21203 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d: No bootstrap required, opened a new log
14:14:36.165 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:14:36.165 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
14:14:36.166 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
14:14:36.166 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
14:14:36.167 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.20.113.126"
port: 44003
}
bound_http_address {
host: "127.20.113.126"
port: 45853
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.20.113.125"
port: 43287
}
bound_http_address {
host: "127.20.113.125"
port: 37077
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.20.113.124"
port: 34873
}
bound_http_address {
host: "127.20.113.124"
port: 40487
}
}
}
14:14:36.167 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
14:14:36.167 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:14:36.168 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
14:14:36.168 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
14:14:36.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.184844 21203 raft_consensus.cc:359] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } }
14:14:36.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.185576 21203 raft_consensus.cc:385] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:14:36.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.185871 21203 raft_consensus.cc:740] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 661b46d073c04359b178727bdbe67c4d, State: Initialized, Role: FOLLOWER
14:14:36.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.186743 21203 consensus_queue.cc:260] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } }
14:14:36.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.189461 21214 sys_catalog.cc:455] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } } }
14:14:36.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.190166 21214 sys_catalog.cc:458] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [sys.catalog]: This master's current role is: FOLLOWER
14:14:36.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.191812 21203 sys_catalog.cc:565] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [sys.catalog]: configured and running, proceeding with master startup.
14:14:36.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 25 14:14:36 dist-test-slave-bg5s krb5kdc[20944](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761401676, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:14:36.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
14:14:36.202 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:14:36.203 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/krb5cc
14:14:36.205 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
14:14:36.205 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/krb5cc
14:14:36.206 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
14:14:36.206 [DEBUG - Time-limited test] (SecurityContext.java:158) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
14:14:36.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:36.218936 21225 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:14:36.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:36.219296 21225 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:14:36.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:36.234828 21155 tablet.cc:2378] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:14:36.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.215487 21124 master_runner.cc:186] Error getting master registration for 127.20.113.126:44003: OK, instance_id { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" instance_seqno: 1761401671666911 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:14:36.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.235536 21124 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:14:36.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.336207 21214 raft_consensus.cc:493] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:14:36.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.336829 21214 raft_consensus.cc:515] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } }
14:14:36.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.339826 21214 leader_election.cc:290] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f7fa589d2c08468aa6cd09db9c033551 (127.20.113.126:44003), 70c2c7e4f63b4d3f94198427af0a5a3e (127.20.113.125:43287)
14:14:36.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.398056 21031 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } attempt: 7
14:14:36.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 25 14:14:36 dist-test-slave-bg5s krb5kdc[20944](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761401671, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.113.126@KRBTEST.COM for kudu/127.20.113.124@KRBTEST.COM
14:14:36.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.429177 21031 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551: Bootstrap starting.
14:14:36.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.436105 21031 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551: Neither blocks nor log segments found. Creating new log.
14:14:36.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.437963 21031 log.cc:826] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551: Log is configured to *not* fsync() on all Append() calls
14:14:36.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.442452 21031 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551: No bootstrap required, opened a new log
14:14:36.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.467274 21031 raft_consensus.cc:359] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } }
14:14:36.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.468072 21031 raft_consensus.cc:385] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:14:36.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.468335 21031 raft_consensus.cc:740] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f7fa589d2c08468aa6cd09db9c033551, State: Initialized, Role: FOLLOWER
14:14:36.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.469228 21031 consensus_queue.cc:260] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } }
14:14:36.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.472092 21229 sys_catalog.cc:455] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } } }
14:14:36.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.472834 21229 sys_catalog.cc:458] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [sys.catalog]: This master's current role is: FOLLOWER
14:14:36.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.473892 21031 sys_catalog.cc:565] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [sys.catalog]: configured and running, proceeding with master startup.
14:14:36.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.475688 21006 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "661b46d073c04359b178727bdbe67c4d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f7fa589d2c08468aa6cd09db9c033551" is_pre_election: true
14:14:36.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.478421 21006 raft_consensus.cc:2468] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 661b46d073c04359b178727bdbe67c4d in term 0.
14:14:36.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.480142 21150 leader_election.cc:304] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 661b46d073c04359b178727bdbe67c4d, f7fa589d2c08468aa6cd09db9c033551; no voters:
14:14:36.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.481079 21214 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [term 0 FOLLOWER]: Leader pre-election won for term 1
14:14:36.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.481402 21214 raft_consensus.cc:493] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:14:36.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.481698 21214 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [term 0 FOLLOWER]: Advancing to term 1
14:14:36.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.489068 21214 raft_consensus.cc:515] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } }
14:14:36.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.491813 21006 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "661b46d073c04359b178727bdbe67c4d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f7fa589d2c08468aa6cd09db9c033551"
14:14:36.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.492301 21006 raft_consensus.cc:3060] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [term 0 FOLLOWER]: Advancing to term 1
14:14:36.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.499369 21006 raft_consensus.cc:2468] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 661b46d073c04359b178727bdbe67c4d in term 1.
14:14:36.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.499864 21214 leader_election.cc:290] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [CANDIDATE]: Term 1 election: Requested vote from peers f7fa589d2c08468aa6cd09db9c033551 (127.20.113.126:44003), 70c2c7e4f63b4d3f94198427af0a5a3e (127.20.113.125:43287)
14:14:36.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.500663 21150 leader_election.cc:304] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 661b46d073c04359b178727bdbe67c4d, f7fa589d2c08468aa6cd09db9c033551; no voters:
14:14:36.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.501960 21214 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [term 1 FOLLOWER]: Leader election won for term 1
14:14:36.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.502365 21214 raft_consensus.cc:697] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [term 1 LEADER]: Becoming Leader. State: Replica: 661b46d073c04359b178727bdbe67c4d, State: Running, Role: LEADER
14:14:36.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.503296 21214 consensus_queue.cc:237] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } }
14:14:36.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.524013 21240 sys_catalog.cc:455] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 661b46d073c04359b178727bdbe67c4d. Latest consensus state: current_term: 1 leader_uuid: "661b46d073c04359b178727bdbe67c4d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } } }
14:14:36.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.525394 21240 sys_catalog.cc:458] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [sys.catalog]: This master's current role is: LEADER
14:14:36.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:36.546914 21241 catalog_manager.cc:1568] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:14:36.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:36.547240 21241 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:14:36.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.551652 21243 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
14:14:36.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.569854 21243 catalog_manager.cc:1494] Initializing Kudu cluster ID...
14:14:36.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.516692 20961 master_runner.cc:186] Error getting master registration for 127.20.113.125:43287: OK, instance_id { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" instance_seqno: 1761401673607029 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:14:36.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.574832 20961 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:14:36.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.588137 21006 raft_consensus.cc:1275] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [term 1 FOLLOWER]: Refusing update from remote peer 661b46d073c04359b178727bdbe67c4d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:14:36.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.591889 21214 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [LEADER]: Connected to new peer: Peer: permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:14:36.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.617455 21229 sys_catalog.cc:455] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 661b46d073c04359b178727bdbe67c4d. Latest consensus state: current_term: 1 leader_uuid: "661b46d073c04359b178727bdbe67c4d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } } }
14:14:36.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.618197 21229 sys_catalog.cc:458] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [sys.catalog]: This master's current role is: FOLLOWER
14:14:36.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.636466 21229 sys_catalog.cc:455] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "661b46d073c04359b178727bdbe67c4d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } } }
14:14:36.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.637332 21229 sys_catalog.cc:458] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [sys.catalog]: This master's current role is: FOLLOWER
14:14:36.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.637216 21246 mvcc.cc:204] Tried to move back new op lower bound from 7214701267285823488 to 7214701266966884352. Current Snapshot: MvccSnapshot[applied={T|T < 7214701267285823488}]
14:14:36.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.646425 21214 sys_catalog.cc:455] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "661b46d073c04359b178727bdbe67c4d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } } }
14:14:36.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.647392 21214 sys_catalog.cc:458] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [sys.catalog]: This master's current role is: LEADER
14:14:36.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.649971 21243 catalog_manager.cc:1357] Generated new cluster ID: e1b948fcf12a4a879df7c714938a923b
14:14:36.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.650334 21243 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
14:14:36.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.690923 21243 catalog_manager.cc:1380] Generated new certificate authority record
14:14:36.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.692852 21243 catalog_manager.cc:1514] Loading token signing keys...
14:14:36.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.738054 21243 catalog_manager.cc:6022] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d: Generated new TSK 0
14:14:36.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:36.739112 21243 catalog_manager.cc:1524] Initializing in-progress tserver states...
14:14:37.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:37.552702 21241 catalog_manager.cc:1269] Loaded cluster ID: e1b948fcf12a4a879df7c714938a923b
14:14:37.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:37.553196 21241 catalog_manager.cc:1562] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551: loading cluster ID for follower catalog manager: success
14:14:37.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:37.561269 21241 catalog_manager.cc:1584] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551: acquiring CA information for follower catalog manager: success
14:14:37.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:37.566318 21241 catalog_manager.cc:1612] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:14:38.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:38.134225 21114 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } attempt: 8
14:14:38.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 25 14:14:38 dist-test-slave-bg5s krb5kdc[20944](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761401673, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.113.125@KRBTEST.COM for kudu/127.20.113.124@KRBTEST.COM
14:14:38.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:38.184952 21153 proxy.cc:239] Call had error, refreshing address and retrying: Timed out: RequestConsensusVote RPC to 127.20.113.125:43287 timed out after 1.844s (SENT)
14:14:38.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:38.213079 21114 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e: Bootstrap starting.
14:14:38.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:38.220116 21114 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e: Neither blocks nor log segments found. Creating new log.
14:14:38.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:38.224583 21114 log.cc:826] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e: Log is configured to *not* fsync() on all Append() calls
14:14:38.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:38.232399 21114 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e: No bootstrap required, opened a new log
14:14:38.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:38.278163 21114 raft_consensus.cc:359] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } }
14:14:38.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:38.278939 21114 raft_consensus.cc:385] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:14:38.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:38.279201 21114 raft_consensus.cc:740] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 70c2c7e4f63b4d3f94198427af0a5a3e, State: Initialized, Role: FOLLOWER
14:14:38.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:38.280186 21114 consensus_queue.cc:260] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } }
14:14:38.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:38.288478 21258 sys_catalog.cc:455] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } } }
14:14:38.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:38.289139 21258 sys_catalog.cc:458] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [sys.catalog]: This master's current role is: FOLLOWER
14:14:38.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:38.291826 21114 sys_catalog.cc:565] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [sys.catalog]: configured and running, proceeding with master startup.
14:14:38.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:38.292362 21087 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [term 0 FOLLOWER]: Advancing to term 1
14:14:38.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:38.296051 21088 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "661b46d073c04359b178727bdbe67c4d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e"
14:14:38.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:38.297096 21088 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:42232 (request call id 2) took 1796 ms (client timeout 1891 ms). Trace:
14:14:38.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:38.297428 21088 rpcz_store.cc:269] 1025 14:14:36.500547 (+ 0us) service_pool.cc:168] Inserting onto call queue
14:14:38.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:14:36.500660 (+ 113us) service_pool.cc:225] Handling call
14:14:38.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:14:38.297017 (+1796357us) inbound_call.cc:173] Queueing success response
14:14:38.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
14:14:38.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:38.300017 21089 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "661b46d073c04359b178727bdbe67c4d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" is_pre_election: true
14:14:38.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:38.300848 21089 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:42232 (request call id 1) took 1960 ms (client timeout 1844 ms). Trace:
14:14:38.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:38.301131 21089 rpcz_store.cc:269] 1025 14:14:36.340125 (+ 0us) service_pool.cc:168] Inserting onto call queue
14:14:38.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:14:36.340276 (+ 151us) service_pool.cc:225] Handling call
14:14:38.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:14:38.300800 (+1960524us) inbound_call.cc:173] Queueing success response
14:14:38.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
14:14:38.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:38.302558 21086 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "661b46d073c04359b178727bdbe67c4d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" is_pre_election: true
14:14:38.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:38.307741 21087 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [term 1 FOLLOWER]: Refusing update from remote peer 661b46d073c04359b178727bdbe67c4d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:14:38.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:38.308437 21087 rpcz_store.cc:275] Call kudu.consensus.ConsensusService.UpdateConsensus from 127.0.0.1:42232 (request call id 3) took 1721 ms. Trace:
14:14:38.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:38.308781 21087 rpcz_store.cc:276] 1025 14:14:36.586888 (+ 0us) service_pool.cc:168] Inserting onto call queue
14:14:38.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:14:36.587016 (+ 128us) service_pool.cc:225] Handling call
14:14:38.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:14:38.292111 (+1705095us) raft_consensus.cc:1471] Updating replica for 0 ops
14:14:38.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:14:38.308117 (+ 16006us) raft_consensus.cc:1697] Filling consensus response to leader.
14:14:38.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:14:38.308390 (+ 273us) inbound_call.cc:173] Queueing success response
14:14:38.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
14:14:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:38.311848 21252 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [LEADER]: Connected to new peer: Peer: permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:14:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:38.314471 21068 tablet.cc:2378] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:14:38.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:38.385324 21258 sys_catalog.cc:455] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [sys.catalog]: SysCatalogTable state changed. Reason: New leader 661b46d073c04359b178727bdbe67c4d. Latest consensus state: current_term: 1 leader_uuid: "661b46d073c04359b178727bdbe67c4d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } } }
14:14:38.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:38.385674 21264 sys_catalog.cc:455] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "661b46d073c04359b178727bdbe67c4d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } } }
14:14:38.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:38.390832 21258 sys_catalog.cc:458] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [sys.catalog]: This master's current role is: FOLLOWER
14:14:38.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:38.391021 21264 sys_catalog.cc:458] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [sys.catalog]: This master's current role is: FOLLOWER
14:14:38.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:38.418011 21274 catalog_manager.cc:1269] Loaded cluster ID: e1b948fcf12a4a879df7c714938a923b
14:14:38.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:38.418335 21274 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e: loading cluster ID for follower catalog manager: success
14:14:38.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:38.431844 21274 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e: acquiring CA information for follower catalog manager: success
14:14:38.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:38.436676 21274 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:14:38.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:38.939217 21252 sys_catalog.cc:455] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "661b46d073c04359b178727bdbe67c4d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } } }
14:14:38.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:38.939997 21252 sys_catalog.cc:458] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [sys.catalog]: This master's current role is: LEADER
14:14:39.208 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.20.113.126:44003
14:14:39.208 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
14:14:39.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:39.209225 20933 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu with pid 20961
14:14:39.248 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:14:39.249 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.20.113.125:43287
14:14:39.249 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
14:14:39.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:39.249570 20933 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu with pid 21040
14:14:39.277 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:14:39.277 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.20.113.124:34873
14:14:39.278 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
14:14:39.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:39.278263 20933 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu with pid 21124
14:14:39.309 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:14:39.309 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.20.113.126:44003
14:14:39.309 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
14:14:39.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:39.311435 20933 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu
14:14:39.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu
14:14:39.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/wal
14:14:39.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/data
14:14:39.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:14:39.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:14:39.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:14:39.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:39.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:39.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:39.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/logs
14:14:39.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/data/info.pb
14:14:39.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:39.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:14:39.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:14:39.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:14:39.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:14:39.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:39.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:39.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:39.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:14:39.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:14:39.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:14:39.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:14:39.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.113.126:44003
14:14:39.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.113.126
14:14:39.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45853
14:14:39.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kudu.keytab
14:14:39.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.113.126
14:14:39.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:14:39.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:14:39.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:14:39.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:14:39.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.113.84:43553
14:14:39.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:39.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:39.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:14:39.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:14:39.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.113.126:44003,127.20.113.125:43287,127.20.113.124:34873
14:14:39.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:14:39.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33059/jwks.json
14:14:39.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:14:39.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:14:39.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:39.624526 21279 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:14:39.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:39.625159 21279 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:14:39.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:39.625847 21279 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:14:39.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:39.658495 21279 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:14:39.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:39.658740 21279 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:14:39.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:39.659139 21279 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:14:39.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:39.659469 21279 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:14:39.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:39.659754 21279 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:14:39.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:39.660032 21279 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:14:39.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:39.660248 21279 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33059/jwks.json
14:14:39.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:39.697526 21279 master_runner.cc:386] Master server non-default flags:
14:14:39.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:39.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.113.84:43553
14:14:39.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:39.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:14:39.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/data
14:14:39.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/wal
14:14:39.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:14:39.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:14:39.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.113.126:44003,127.20.113.125:43287,127.20.113.124:34873
14:14:39.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kudu.keytab
14:14:39.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.113.126
14:14:39.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:39.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:39.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:14:39.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.113.126:44003
14:14:39.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:14:39.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:14:39.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:14:39.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33059/jwks.json
14:14:39.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:14:39.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:14:39.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:14:39.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:39.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:39.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/data/info.pb
14:14:39.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.113.126
14:14:39.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45853
14:14:39.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:14:39.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:39.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:14:39.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:14:39.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:39.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/logs
14:14:39.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:39.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:14:39.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:14:39.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:14:39.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:14:39.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d0cb013a5aad754f2ed06c25f5ec3d8a1873622
14:14:39.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:14:39.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Oct 2025 13:43:23 UTC on 5fd53c4cbb9d
14:14:39.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8713
14:14:39.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:14:39.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:39.699129 21279 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:14:39.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:39.700433 21279 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:14:39.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:39.712378 21285 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:40.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:39.713742 21286 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:40.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:40.944960 21288 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:40.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:40.947491 21287 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1230 milliseconds
14:14:40.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:40.947567 21279 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:14:40.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:40.948730 21279 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:14:40.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:40.951798 21279 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:14:40.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:40.953200 21279 hybrid_clock.cc:648] HybridClock initialized: now 1761401680953153 us; error 53 us; skew 500 ppm
14:14:40.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:40.953971 21279 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:14:40.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 25 14:14:40 dist-test-slave-bg5s krb5kdc[20944](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761401680, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.113.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:14:40.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:40.959784 21279 init.cc:377] Logged in from keytab as kudu/127.20.113.126@KRBTEST.COM (short username kudu)
14:14:40.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:40.964627 21279 webserver.cc:492] Webserver started at http://127.20.113.126:45853/ using document root <none> and password file <none>
14:14:40.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:40.965498 21279 fs_manager.cc:362] Metadata directory not provided
14:14:40.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:40.965718 21279 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:14:40.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:40.973121 21279 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.001s sys 0.003s
14:14:40.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:40.977448 21297 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:14:40.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:40.978371 21279 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.001s
14:14:40.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:40.978686 21279 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/data,/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/wal
14:14:40.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f7fa589d2c08468aa6cd09db9c033551"
14:14:40.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 14:14:31 on dist-test-slave-bg5s"
14:14:40.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:40.980667 21279 fs_report.cc:389] FS layout report
14:14:40.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:14:40.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/wal
14:14:40.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/wal
14:14:40.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/data/data
14:14:40.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:14:40.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:14:40.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:14:40.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:14:40.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:14:40.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:14:40.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:14:40.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:14:40.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:14:40.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:14:40.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:14:41.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:41.034161 21279 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:14:41.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:41.039863 21279 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:14:41.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:41.040266 21279 kserver.cc:163] Server-wide thread pool size limit: 3276
14:14:41.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:41.105538 21279 rpc_server.cc:307] RPC server started. Bound to: 127.20.113.126:44003
14:14:41.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:41.105599 21348 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.113.126:44003 every 8 connection(s)
14:14:41.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:41.107939 21279 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/data/info.pb
14:14:41.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:41.116490 20933 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu as pid 21279
14:14:41.117 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:14:41.117 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.20.113.125:43287
14:14:41.117 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
14:14:41.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:41.118337 21349 sys_catalog.cc:263] Verifying existing consensus state
14:14:41.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:41.118363 20933 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu
14:14:41.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu
14:14:41.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/wal
14:14:41.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/data
14:14:41.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:14:41.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:14:41.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:14:41.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:41.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:41.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:41.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/logs
14:14:41.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/data/info.pb
14:14:41.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:41.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:14:41.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:14:41.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:14:41.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:14:41.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:41.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:41.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:41.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:14:41.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:14:41.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:14:41.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:14:41.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.113.125:43287
14:14:41.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.113.125
14:14:41.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37077
14:14:41.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kudu.keytab
14:14:41.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.113.125
14:14:41.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:14:41.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:14:41.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:14:41.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:14:41.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.113.84:43553
14:14:41.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:41.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:41.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:14:41.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:14:41.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.113.126:44003,127.20.113.125:43287,127.20.113.124:34873
14:14:41.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:14:41.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33059/jwks.json
14:14:41.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:14:41.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:14:41.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:41.123138 21349 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551: Bootstrap starting.
14:14:41.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:41.159932 21349 log.cc:826] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551: Log is configured to *not* fsync() on all Append() calls
14:14:41.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:41.174114 21349 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:14:41.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:41.175063 21349 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551: Bootstrap complete.
14:14:41.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:41.197068 21349 raft_consensus.cc:359] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } }
14:14:41.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:41.199211 21349 raft_consensus.cc:740] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: f7fa589d2c08468aa6cd09db9c033551, State: Initialized, Role: FOLLOWER
14:14:41.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:41.200001 21349 consensus_queue.cc:260] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } }
14:14:41.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:41.202865 21353 sys_catalog.cc:455] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } } }
14:14:41.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:41.203670 21353 sys_catalog.cc:458] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [sys.catalog]: This master's current role is: FOLLOWER
14:14:41.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:41.204566 21349 sys_catalog.cc:565] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [sys.catalog]: configured and running, proceeding with master startup.
14:14:41.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:41.236713 21364 catalog_manager.cc:1269] Loaded cluster ID: e1b948fcf12a4a879df7c714938a923b
14:14:41.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:41.237058 21364 catalog_manager.cc:1562] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551: loading cluster ID for follower catalog manager: success
14:14:41.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:41.244374 21364 catalog_manager.cc:1584] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551: acquiring CA information for follower catalog manager: success
14:14:41.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 25 14:14:41 dist-test-slave-bg5s krb5kdc[20944](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761401680, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.113.126@KRBTEST.COM for kudu/127.20.113.126@KRBTEST.COM
14:14:41.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:41.249151 21364 catalog_manager.cc:1612] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:14:41.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:41.267302 21279 master_runner.cc:186] Error getting master registration for 127.20.113.125:43287: Network error: Client connection negotiation failed: client connection to 127.20.113.125:43287: connect: Connection refused (error 111),
14:14:41.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:41.292649 21279 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:14:41.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:41.445116 21351 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:14:41.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:41.445691 21351 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:14:41.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:41.446352 21351 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:14:41.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:41.476897 21351 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:14:41.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:41.477169 21351 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:14:41.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:41.477533 21351 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:14:41.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:41.477789 21351 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:14:41.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:41.478036 21351 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:14:41.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:41.478282 21351 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:14:41.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:41.478474 21351 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33059/jwks.json
14:14:41.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:41.514050 21351 master_runner.cc:386] Master server non-default flags:
14:14:41.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:41.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.113.84:43553
14:14:41.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:14:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/data
14:14:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/wal
14:14:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:14:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:14:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.113.126:44003,127.20.113.125:43287,127.20.113.124:34873
14:14:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kudu.keytab
14:14:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.113.125
14:14:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:14:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.113.125:43287
14:14:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:14:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:14:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:14:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33059/jwks.json
14:14:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:14:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:14:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:14:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/data/info.pb
14:14:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.113.125
14:14:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37077
14:14:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:14:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:14:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:14:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/logs
14:14:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:14:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:14:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:14:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:14:41.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d0cb013a5aad754f2ed06c25f5ec3d8a1873622
14:14:41.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:14:41.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Oct 2025 13:43:23 UTC on 5fd53c4cbb9d
14:14:41.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8713
14:14:41.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:14:41.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:41.515509 21351 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:14:41.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:41.516697 21351 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:14:41.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:41.530704 21374 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:42.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:41.532702 21373 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:42.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:42.927284 21351 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.400s user 0.509s sys 0.835s
14:14:42.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:42.928382 21351 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.401s user 0.511s sys 0.837s
14:14:42.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:42.930126 21378 raft_consensus.cc:493] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:14:42.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:42.930776 21378 raft_consensus.cc:515] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } }
14:14:42.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:42.940114 21301 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.113.125:43287: connect: Connection refused (error 111)
14:14:42.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:42.947340 21351 server_base.cc:1047] running on GCE node
14:14:42.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:42.947284 21378 leader_election.cc:290] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 70c2c7e4f63b4d3f94198427af0a5a3e (127.20.113.125:43287), 661b46d073c04359b178727bdbe67c4d (127.20.113.124:34873)
14:14:42.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:42.950404 21380 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:42.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:42.951216 21301 leader_election.cc:336] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 70c2c7e4f63b4d3f94198427af0a5a3e (127.20.113.125:43287): Network error: Client connection negotiation failed: client connection to 127.20.113.125:43287: connect: Connection refused (error 111)
14:14:42.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:42.953562 21351 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:14:42.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:42.964501 21299 leader_election.cc:336] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 661b46d073c04359b178727bdbe67c4d (127.20.113.124:34873): Network error: Client connection negotiation failed: client connection to 127.20.113.124:34873: connect: Connection refused (error 111)
14:14:42.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:42.965093 21299 leader_election.cc:304] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: f7fa589d2c08468aa6cd09db9c033551; no voters: 661b46d073c04359b178727bdbe67c4d, 70c2c7e4f63b4d3f94198427af0a5a3e
14:14:42.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:42.966115 21378 raft_consensus.cc:2749] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
14:14:42.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:42.968055 21351 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:14:42.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:42.969467 21351 hybrid_clock.cc:648] HybridClock initialized: now 1761401682969411 us; error 70 us; skew 500 ppm
14:14:42.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:42.970366 21351 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:14:42.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 25 14:14:42 dist-test-slave-bg5s krb5kdc[20944](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761401682, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.113.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:14:42.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:42.977223 21351 init.cc:377] Logged in from keytab as kudu/127.20.113.125@KRBTEST.COM (short username kudu)
14:14:42.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:42.984171 21351 webserver.cc:492] Webserver started at http://127.20.113.125:37077/ using document root <none> and password file <none>
14:14:42.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:42.985160 21351 fs_manager.cc:362] Metadata directory not provided
14:14:42.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:42.985441 21351 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:14:42.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:42.993643 21351 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.004s sys 0.001s
14:14:43.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:43.000255 21389 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:14:43.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:43.001391 21351 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.001s sys 0.002s
14:14:43.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:43.001724 21351 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/data,/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/wal
14:14:43.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "70c2c7e4f63b4d3f94198427af0a5a3e"
14:14:43.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 14:14:33 on dist-test-slave-bg5s"
14:14:43.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:43.003592 21351 fs_report.cc:389] FS layout report
14:14:43.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:14:43.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/wal
14:14:43.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/wal
14:14:43.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/data/data
14:14:43.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:14:43.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:14:43.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:14:43.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:14:43.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:14:43.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:14:43.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:14:43.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:14:43.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:14:43.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:14:43.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:14:43.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:43.083937 21351 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:14:43.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:43.089113 21351 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:14:43.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:43.089548 21351 kserver.cc:163] Server-wide thread pool size limit: 3276
14:14:43.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:43.178145 21351 rpc_server.cc:307] RPC server started. Bound to: 127.20.113.125:43287
14:14:43.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:43.178216 21443 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.113.125:43287 every 8 connection(s)
14:14:43.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:43.181967 21351 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/data/info.pb
14:14:43.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:43.188882 20933 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu as pid 21351
14:14:43.189 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:14:43.189 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.20.113.124:34873
14:14:43.189 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
14:14:43.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:43.191394 20933 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu
14:14:43.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu
14:14:43.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/wal
14:14:43.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/data
14:14:43.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:14:43.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:14:43.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:14:43.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:43.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:43.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:43.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/logs
14:14:43.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/data/info.pb
14:14:43.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:43.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:14:43.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:14:43.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:14:43.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:14:43.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:43.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:43.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:43.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:14:43.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:14:43.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:14:43.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:14:43.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.113.124:34873
14:14:43.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.113.124
14:14:43.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40487
14:14:43.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kudu.keytab
14:14:43.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.113.124
14:14:43.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:14:43.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:14:43.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:14:43.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:14:43.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.113.84:43553
14:14:43.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:43.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:43.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:14:43.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:14:43.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.113.126:44003,127.20.113.125:43287,127.20.113.124:34873
14:14:43.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:14:43.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33059/jwks.json
14:14:43.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:14:43.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:14:43.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:43.194463 21444 sys_catalog.cc:263] Verifying existing consensus state
14:14:43.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:43.201727 21444 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e: Bootstrap starting.
14:14:43.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:43.252858 21444 log.cc:826] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e: Log is configured to *not* fsync() on all Append() calls
14:14:43.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:43.277849 21444 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:14:43.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:43.278836 21444 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e: Bootstrap complete.
14:14:43.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:43.312433 21444 raft_consensus.cc:359] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } }
14:14:43.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:43.315966 21444 raft_consensus.cc:740] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 70c2c7e4f63b4d3f94198427af0a5a3e, State: Initialized, Role: FOLLOWER
14:14:43.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:43.317050 21444 consensus_queue.cc:260] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } }
14:14:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:43.321223 21448 sys_catalog.cc:455] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } } }
14:14:43.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:43.322125 21448 sys_catalog.cc:458] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [sys.catalog]: This master's current role is: FOLLOWER
14:14:43.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:43.323158 21444 sys_catalog.cc:565] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [sys.catalog]: configured and running, proceeding with master startup.
14:14:43.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:43.376163 21460 catalog_manager.cc:1269] Loaded cluster ID: e1b948fcf12a4a879df7c714938a923b
14:14:43.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:43.376634 21460 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e: loading cluster ID for follower catalog manager: success
14:14:43.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:43.388381 21460 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e: acquiring CA information for follower catalog manager: success
14:14:43.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:43.393851 21460 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:14:43.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 25 14:14:43 dist-test-slave-bg5s krb5kdc[20944](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761401682, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.113.125@KRBTEST.COM for kudu/127.20.113.126@KRBTEST.COM
14:14:43.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:43.465266 21351 master_runner.cc:186] Error getting master registration for 127.20.113.124:34873: Network error: Client connection negotiation failed: client connection to 127.20.113.124:34873: connect: Connection refused (error 111),
14:14:43.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:43.491235 21351 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:14:43.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:43.619496 21446 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:14:43.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:43.620141 21446 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:14:43.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:43.620836 21446 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:14:43.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:43.652722 21446 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:14:43.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:43.652964 21446 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:14:43.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:43.653345 21446 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:14:43.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:43.653599 21446 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:14:43.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:43.653970 21446 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:14:43.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:43.654227 21446 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:14:43.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:43.654424 21446 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33059/jwks.json
14:14:43.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:43.700562 21446 master_runner.cc:386] Master server non-default flags:
14:14:43.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:43.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.113.84:43553
14:14:43.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:43.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:14:43.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/data
14:14:43.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/wal
14:14:43.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:14:43.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:14:43.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.113.126:44003,127.20.113.125:43287,127.20.113.124:34873
14:14:43.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kudu.keytab
14:14:43.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.113.124
14:14:43.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:43.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:43.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:14:43.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.113.124:34873
14:14:43.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:14:43.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:14:43.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:14:43.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33059/jwks.json
14:14:43.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:14:43.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:14:43.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:14:43.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:43.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:43.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/data/info.pb
14:14:43.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.113.124
14:14:43.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40487
14:14:43.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:14:43.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:43.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:14:43.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:14:43.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:43.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/logs
14:14:43.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:43.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:14:43.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:14:43.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:14:43.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:14:43.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d0cb013a5aad754f2ed06c25f5ec3d8a1873622
14:14:43.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:14:43.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Oct 2025 13:43:23 UTC on 5fd53c4cbb9d
14:14:43.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8713
14:14:43.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:14:43.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:43.702154 21446 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:14:43.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:43.703233 21446 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:14:43.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:43.735311 21470 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:44.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:44.835408 21476 raft_consensus.cc:493] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:14:44.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:44.836304 21476 raft_consensus.cc:515] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } }
14:14:44.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:44.845048 21476 leader_election.cc:290] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 70c2c7e4f63b4d3f94198427af0a5a3e (127.20.113.125:43287), 661b46d073c04359b178727bdbe67c4d (127.20.113.124:34873)
14:14:44.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:44.874467 21419 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f7fa589d2c08468aa6cd09db9c033551" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" is_pre_election: true
14:14:44.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:44.875516 21419 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f7fa589d2c08468aa6cd09db9c033551 in term 1.
14:14:44.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:44.876888 21301 leader_election.cc:304] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 70c2c7e4f63b4d3f94198427af0a5a3e, f7fa589d2c08468aa6cd09db9c033551; no voters:
14:14:44.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:44.877967 21476 raft_consensus.cc:2804] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [term 1 FOLLOWER]: Leader pre-election won for term 2
14:14:44.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:44.878332 21476 raft_consensus.cc:493] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:14:44.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:44.878710 21476 raft_consensus.cc:3060] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [term 1 FOLLOWER]: Advancing to term 2
14:14:44.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:44.892747 21299 leader_election.cc:336] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 661b46d073c04359b178727bdbe67c4d (127.20.113.124:34873): Network error: Client connection negotiation failed: client connection to 127.20.113.124:34873: connect: Connection refused (error 111)
14:14:44.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:44.891355 21476 raft_consensus.cc:515] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } }
14:14:44.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:44.909004 21419 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f7fa589d2c08468aa6cd09db9c033551" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e"
14:14:44.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:44.909519 21419 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [term 1 FOLLOWER]: Advancing to term 2
14:14:44.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:44.912194 21299 leader_election.cc:336] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 661b46d073c04359b178727bdbe67c4d (127.20.113.124:34873): Network error: Client connection negotiation failed: client connection to 127.20.113.124:34873: connect: Connection refused (error 111)
14:14:44.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:44.910308 21476 leader_election.cc:290] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [CANDIDATE]: Term 2 election: Requested vote from peers 70c2c7e4f63b4d3f94198427af0a5a3e (127.20.113.125:43287), 661b46d073c04359b178727bdbe67c4d (127.20.113.124:34873)
14:14:44.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:44.924001 21419 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f7fa589d2c08468aa6cd09db9c033551 in term 2.
14:14:44.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:44.928674 21301 leader_election.cc:304] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 70c2c7e4f63b4d3f94198427af0a5a3e, f7fa589d2c08468aa6cd09db9c033551; no voters: 661b46d073c04359b178727bdbe67c4d
14:14:44.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:44.929718 21476 raft_consensus.cc:2804] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [term 2 FOLLOWER]: Leader election won for term 2
14:14:44.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:44.950645 21476 raft_consensus.cc:697] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [term 2 LEADER]: Becoming Leader. State: Replica: f7fa589d2c08468aa6cd09db9c033551, State: Running, Role: LEADER
14:14:44.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:44.952123 21476 consensus_queue.cc:237] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } }
14:14:44.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:44.976583 21481 sys_catalog.cc:455] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f7fa589d2c08468aa6cd09db9c033551. Latest consensus state: current_term: 2 leader_uuid: "f7fa589d2c08468aa6cd09db9c033551" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } } }
14:14:44.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:44.977872 21481 sys_catalog.cc:458] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [sys.catalog]: This master's current role is: LEADER
14:14:44.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:44.989198 21483 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
14:14:44.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:44.998767 21483 catalog_manager.cc:1494] Initializing Kudu cluster ID...
14:14:45.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:45.015910 21483 catalog_manager.cc:1269] Loaded cluster ID: e1b948fcf12a4a879df7c714938a923b
14:14:45.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:45.016438 21483 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
14:14:45.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:45.034245 21483 catalog_manager.cc:1514] Loading token signing keys...
14:14:45.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:45.039181 21483 catalog_manager.cc:6033] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551: Loaded TSK: 0
14:14:45.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:45.040786 21483 catalog_manager.cc:1524] Initializing in-progress tserver states...
14:14:45.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:45.380506 21299 consensus_peers.cc:597] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 -> Peer 661b46d073c04359b178727bdbe67c4d (127.20.113.124:34873): Couldn't send request to peer 661b46d073c04359b178727bdbe67c4d. Status: Network error: Client connection negotiation failed: client connection to 127.20.113.124:34873: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:14:45.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:45.508251 21419 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [term 2 FOLLOWER]: Refusing update from remote peer f7fa589d2c08468aa6cd09db9c033551: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
14:14:45.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:45.511078 21481 consensus_queue.cc:1048] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [LEADER]: Connected to new peer: Peer: permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
14:14:45.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:45.565191 21485 sys_catalog.cc:455] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [sys.catalog]: SysCatalogTable state changed. Reason: New leader f7fa589d2c08468aa6cd09db9c033551. Latest consensus state: current_term: 2 leader_uuid: "f7fa589d2c08468aa6cd09db9c033551" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } } }
14:14:45.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:45.569880 21485 sys_catalog.cc:458] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [sys.catalog]: This master's current role is: FOLLOWER
14:14:45.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:45.591595 21485 sys_catalog.cc:455] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "f7fa589d2c08468aa6cd09db9c033551" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } } }
14:14:45.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:45.592561 21485 sys_catalog.cc:458] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [sys.catalog]: This master's current role is: FOLLOWER
14:14:45.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:45.594116 21481 sys_catalog.cc:455] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "f7fa589d2c08468aa6cd09db9c033551" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } } }
14:14:45.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:45.594870 21481 sys_catalog.cc:458] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [sys.catalog]: This master's current role is: LEADER
14:14:45.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:45.134320 21469 debug-util.cc:398] Leaking SignalData structure 0x7b0800027080 after lost signal to thread 21446
14:14:45.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:45.666689 21469 kernel_stack_watchdog.cc:198] Thread 21446 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 395ms:
14:14:45.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:14:45.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:14:45.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:14:45.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:14:45.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:45.667495 21446 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.931s user 0.524s sys 0.978s
14:14:45.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:45.667819 21446 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.931s user 0.524s sys 0.978s
14:14:45.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:45.668702 21472 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1932 milliseconds
14:14:45.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:43.740274 21471 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:45.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:45.669692 21473 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:45.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:45.670233 21446 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:14:45.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:45.674185 21446 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:14:45.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:45.679013 21446 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:14:45.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:45.680528 21446 hybrid_clock.cc:648] HybridClock initialized: now 1761401685680487 us; error 50 us; skew 500 ppm
14:14:45.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:45.681365 21446 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:14:45.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 25 14:14:45 dist-test-slave-bg5s krb5kdc[20944](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761401685, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.113.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:14:45.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:45.687448 21446 init.cc:377] Logged in from keytab as kudu/127.20.113.124@KRBTEST.COM (short username kudu)
14:14:45.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:45.693429 21446 webserver.cc:492] Webserver started at http://127.20.113.124:40487/ using document root <none> and password file <none>
14:14:45.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:45.694545 21446 fs_manager.cc:362] Metadata directory not provided
14:14:45.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:45.694849 21446 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:14:45.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:45.704545 21446 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.002s sys 0.006s
14:14:45.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:45.710170 21498 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:14:45.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:45.711452 21446 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
14:14:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:45.711799 21446 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/data,/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/wal
14:14:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "661b46d073c04359b178727bdbe67c4d"
14:14:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 14:14:35 on dist-test-slave-bg5s"
14:14:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:45.713997 21446 fs_report.cc:389] FS layout report
14:14:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:14:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/wal
14:14:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/wal
14:14:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/data/data
14:14:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:14:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:14:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:14:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:14:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:14:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:14:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:14:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:14:45.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:14:45.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:14:45.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:14:45.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:45.766418 21446 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:14:45.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:45.775580 21446 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:14:45.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:45.776098 21446 kserver.cc:163] Server-wide thread pool size limit: 3276
14:14:45.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:45.887460 21446 rpc_server.cc:307] RPC server started. Bound to: 127.20.113.124:34873
14:14:45.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:45.887514 21550 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.113.124:34873 every 8 connection(s)
14:14:45.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:45.890141 21446 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/data/info.pb
14:14:45.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:45.898502 20933 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu as pid 21446
14:14:45.899 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:14:45.900 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.20.113.126:44003(127.20.113.126:44003)] connecting to peer
14:14:45.901 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.20.113.125:43287(127.20.113.125:43287)] connecting to peer
14:14:45.902 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.20.113.124:34873(127.20.113.124:34873)] connecting to peer
14:14:45.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:45.903067 21551 sys_catalog.cc:263] Verifying existing consensus state
14:14:45.912 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.20.113.125:43287(127.20.113.125:43287)] Successfully connected to peer
14:14:45.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:45.916983 21551 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d: Bootstrap starting.
14:14:45.918 [WARN - kudu-nio-1] (SecurityContext.java:206) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
14:14:45.918 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.20.113.126:44003(127.20.113.126:44003)] Successfully connected to peer
14:14:45.924 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.20.113.125
14:14:45.932 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.20.113.124:34873(127.20.113.124:34873)] Successfully connected to peer
14:14:45.945 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.20.113.126
14:14:45.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 25 14:14:45 dist-test-slave-bg5s krb5kdc[20944](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761401676, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.20.113.125@KRBTEST.COM
14:14:45.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 25 14:14:45 dist-test-slave-bg5s krb5kdc[20944](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761401676, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.20.113.126@KRBTEST.COM
14:14:45.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:45.980822 21551 log.cc:826] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d: Log is configured to *not* fsync() on all Append() calls
14:14:45.987 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xbfcbad2c, L:/127.0.0.1:59328 - R:/127.20.113.125:43287] using SASL/GSSAPI
14:14:45.988 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.20.113.124
14:14:46.008 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.113.124:34873(127.20.113.124:34873)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
14:14:46.009 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.113.124:34873: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
14:14:46.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:46.011133 21551 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:14:46.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:46.012253 21551 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d: Bootstrap complete.
14:14:46.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:46.012741 21558 negotiation.cc:337] Failed RPC negotiation. Trace:
14:14:46.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:14:45.919298 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:39244 (local address 127.20.113.124:34873)
14:14:46.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:14:45.922523 (+ 3225us) server_negotiation.cc:207] Beginning negotiation
14:14:46.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:14:45.922555 (+ 32us) server_negotiation.cc:400] Waiting for connection header
14:14:46.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:14:45.933894 (+ 11339us) server_negotiation.cc:408] Connection header received
14:14:46.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:14:45.945917 (+ 12023us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
14:14:46.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:14:45.945966 (+ 49us) server_negotiation.cc:462] Received NEGOTIATE request from client
14:14:46.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:14:45.946440 (+ 474us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
14:14:46.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:14:45.946874 (+ 434us) server_negotiation.cc:227] Negotiated authn=SASL
14:14:46.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:14:45.992995 (+ 46121us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
14:14:46.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:14:45.995472 (+ 2477us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
14:14:46.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:14:46.007133 (+ 11661us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
14:14:46.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:14:46.007813 (+ 680us) server_negotiation.cc:663] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
14:14:46.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:14:46.010854 (+ 3041us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:39244: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:39244): Cannot send after transport endpoint shutdown (error 108)
14:14:46.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":2553,"thread_start_us":2261,"threads_started":1}
14:14:46.016 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.20.113.124:34873(127.20.113.124:34873)] handling channelInactive
14:14:46.018 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x419d1905, L:/127.0.0.1:51280 - R:/127.20.113.126:44003] using SASL/GSSAPI
14:14:46.032 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
14:14:46.032 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=133), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.113.126:44003, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.113.125:43287, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.113.124:34873, [91ms] Sub RPC ConnectToMaster: received response from server master-127.20.113.125:43287: OK, [110ms] Sub RPC ConnectToMaster: received response from server master-127.20.113.124:34873: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [126ms] Sub RPC ConnectToMaster: received response from server master-127.20.113.126:44003: OK, deferred=null) after lookup
14:14:46.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:46.039043 21551 raft_consensus.cc:359] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } }
14:14:46.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:46.042330 21551 raft_consensus.cc:740] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 661b46d073c04359b178727bdbe67c4d, State: Initialized, Role: FOLLOWER
14:14:46.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:46.043210 21551 consensus_queue.cc:260] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } }
14:14:46.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:46.048401 21560 sys_catalog.cc:455] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } } }
14:14:46.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:46.049216 21560 sys_catalog.cc:458] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [sys.catalog]: This master's current role is: FOLLOWER
14:14:46.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:46.050135 21551 sys_catalog.cc:565] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [sys.catalog]: configured and running, proceeding with master startup.
14:14:46.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:46.081702 21571 catalog_manager.cc:1269] Loaded cluster ID: e1b948fcf12a4a879df7c714938a923b
14:14:46.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:46.082145 21571 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d: loading cluster ID for follower catalog manager: success
14:14:46.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 25 14:14:46 dist-test-slave-bg5s krb5kdc[20944](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761401685, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.113.124@KRBTEST.COM for kudu/127.20.113.126@KRBTEST.COM
14:14:46.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:46.090974 21571 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d: acquiring CA information for follower catalog manager: success
14:14:46.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:46.096694 21571 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:14:46.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:46.252480 21526 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [term 1 FOLLOWER]: Advancing to term 2
14:14:46.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:46.271152 21577 sys_catalog.cc:455] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "f7fa589d2c08468aa6cd09db9c033551" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } } }
14:14:46.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:46.272441 21577 sys_catalog.cc:458] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [sys.catalog]: This master's current role is: FOLLOWER
14:14:46.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:46.276491 21560 sys_catalog.cc:455] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [sys.catalog]: SysCatalogTable state changed. Reason: New leader f7fa589d2c08468aa6cd09db9c033551. Latest consensus state: current_term: 2 leader_uuid: "f7fa589d2c08468aa6cd09db9c033551" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } } }
14:14:46.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:46.277164 21560 sys_catalog.cc:458] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [sys.catalog]: This master's current role is: FOLLOWER
14:14:46.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:46.820143 21481 sys_catalog.cc:455] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "f7fa589d2c08468aa6cd09db9c033551" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } } }
14:14:46.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:46.820885 21481 sys_catalog.cc:458] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [sys.catalog]: This master's current role is: LEADER
14:14:49.086 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.20.113.126:44003
14:14:49.087 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
14:14:49.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:49.087728 20933 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu with pid 21279
14:14:49.115 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.20.113.126:44003(127.20.113.126:44003)] handling channelInactive
14:14:49.116 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.113.126:44003(127.20.113.126:44003)] cleaning up while in state READY due to: connection closed
14:14:49.116 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:14:49.116 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.20.113.125:43287
14:14:49.116 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
14:14:49.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:49.117002 20933 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu with pid 21351
14:14:49.145 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:14:49.145 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.20.113.125:43287(127.20.113.125:43287)] handling channelInactive
14:14:49.145 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.20.113.124:34873
14:14:49.145 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.113.125:43287(127.20.113.125:43287)] cleaning up while in state READY due to: connection closed
14:14:49.146 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
14:14:49.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:49.146246 20933 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu with pid 21446
14:14:49.173 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:14:49.173 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.20.113.126:44003
14:14:49.173 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
14:14:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:49.174770 20933 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu
14:14:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu
14:14:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/wal
14:14:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/data
14:14:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:14:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:14:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:14:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/logs
14:14:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/data/info.pb
14:14:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:14:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:14:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:14:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:14:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:14:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:14:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:14:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:14:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.113.126:44003
14:14:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.113.126
14:14:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45853
14:14:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kudu.keytab
14:14:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.113.126
14:14:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:14:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:14:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:14:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:14:49.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.113.84:43553
14:14:49.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:49.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:49.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:14:49.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:14:49.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.113.126:44003,127.20.113.125:43287,127.20.113.124:34873
14:14:49.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:14:49.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33059/jwks.json
14:14:49.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:14:49.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:14:49.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:49.494174 21585 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:14:49.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:49.494788 21585 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:14:49.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:49.495473 21585 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:14:49.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:49.526533 21585 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:14:49.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:49.526769 21585 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:14:49.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:49.527125 21585 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:14:49.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:49.527403 21585 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:14:49.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:49.527689 21585 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:14:49.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:49.527969 21585 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:14:49.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:49.528179 21585 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33059/jwks.json
14:14:49.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:49.564430 21585 master_runner.cc:386] Master server non-default flags:
14:14:49.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:49.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.113.84:43553
14:14:49.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:49.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:14:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/data
14:14:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/wal
14:14:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:14:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:14:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.113.126:44003,127.20.113.125:43287,127.20.113.124:34873
14:14:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kudu.keytab
14:14:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.113.126
14:14:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:14:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.113.126:44003
14:14:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:14:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:14:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:14:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33059/jwks.json
14:14:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:14:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:14:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:14:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/data/info.pb
14:14:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.113.126
14:14:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45853
14:14:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:14:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:14:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:14:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/logs
14:14:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:14:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:14:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:14:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:14:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d0cb013a5aad754f2ed06c25f5ec3d8a1873622
14:14:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:14:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Oct 2025 13:43:23 UTC on 5fd53c4cbb9d
14:14:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8713
14:14:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:14:49.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:49.565948 21585 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:14:49.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:49.566879 21585 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:14:49.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:49.577319 21591 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:51.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:50.985641 21590 debug-util.cc:398] Leaking SignalData structure 0x7b0800027080 after lost signal to thread 21585
14:14:51.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:51.281381 21590 kernel_stack_watchdog.cc:198] Thread 21585 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 404ms:
14:14:51.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:14:51.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:14:51.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:14:51.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:14:51.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:49.577792 21592 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:51.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:51.282649 21585 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.704s user 0.547s sys 1.131s
14:14:51.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:51.283993 21585 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.706s user 0.547s sys 1.131s
14:14:51.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:51.284428 21594 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:51.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:51.286402 21593 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1703 milliseconds
14:14:51.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:51.286418 21585 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:14:51.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:51.287850 21585 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:14:51.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:51.290051 21585 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:14:51.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:51.291488 21585 hybrid_clock.cc:648] HybridClock initialized: now 1761401691291464 us; error 68 us; skew 500 ppm
14:14:51.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:51.292204 21585 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:14:51.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 25 14:14:51 dist-test-slave-bg5s krb5kdc[20944](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761401691, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.113.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:14:51.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:51.298695 21585 init.cc:377] Logged in from keytab as kudu/127.20.113.126@KRBTEST.COM (short username kudu)
14:14:51.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:51.303792 21585 webserver.cc:492] Webserver started at http://127.20.113.126:45853/ using document root <none> and password file <none>
14:14:51.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:51.304870 21585 fs_manager.cc:362] Metadata directory not provided
14:14:51.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:51.305145 21585 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:14:51.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:51.314422 21585 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.003s sys 0.003s
14:14:51.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:51.318637 21603 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:14:51.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:51.319590 21585 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.001s
14:14:51.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:51.319883 21585 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/data,/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/wal
14:14:51.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f7fa589d2c08468aa6cd09db9c033551"
14:14:51.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 14:14:31 on dist-test-slave-bg5s"
14:14:51.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:51.321902 21585 fs_report.cc:389] FS layout report
14:14:51.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:14:51.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/wal
14:14:51.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/wal
14:14:51.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/data/data
14:14:51.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:14:51.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:14:51.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:14:51.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:14:51.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:14:51.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:14:51.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:14:51.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:14:51.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:14:51.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:14:51.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:14:51.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:51.371563 21585 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:14:51.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:51.379916 21585 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:14:51.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:51.380437 21585 kserver.cc:163] Server-wide thread pool size limit: 3276
14:14:51.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:51.450469 21585 rpc_server.cc:307] RPC server started. Bound to: 127.20.113.126:44003
14:14:51.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:51.450604 21655 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.113.126:44003 every 8 connection(s)
14:14:51.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:51.453284 21585 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/data/info.pb
14:14:51.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:51.461484 20933 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu as pid 21585
14:14:51.462 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:14:51.462 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.20.113.125:43287
14:14:51.462 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
14:14:51.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:51.464030 20933 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu
14:14:51.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu
14:14:51.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/wal
14:14:51.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/data
14:14:51.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:14:51.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:14:51.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:14:51.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:51.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:51.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:51.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/logs
14:14:51.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/data/info.pb
14:14:51.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:51.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:14:51.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:14:51.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:14:51.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:14:51.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:51.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:51.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:51.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:14:51.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:14:51.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:14:51.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:14:51.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.113.125:43287
14:14:51.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.113.125
14:14:51.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37077
14:14:51.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kudu.keytab
14:14:51.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.113.125
14:14:51.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:14:51.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:14:51.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:14:51.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:14:51.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.113.84:43553
14:14:51.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:51.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:51.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:14:51.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:14:51.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.113.126:44003,127.20.113.125:43287,127.20.113.124:34873
14:14:51.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:14:51.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33059/jwks.json
14:14:51.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:14:51.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:14:51.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:51.464362 21656 sys_catalog.cc:263] Verifying existing consensus state
14:14:51.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:51.471447 21656 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551: Bootstrap starting.
14:14:51.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:51.508072 21656 log.cc:826] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551: Log is configured to *not* fsync() on all Append() calls
14:14:51.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:51.523252 21656 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:14:51.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:51.523990 21656 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551: Bootstrap complete.
14:14:51.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:51.544961 21656 raft_consensus.cc:359] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } }
14:14:51.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:51.547019 21656 raft_consensus.cc:740] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: f7fa589d2c08468aa6cd09db9c033551, State: Initialized, Role: FOLLOWER
14:14:51.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:51.547821 21656 consensus_queue.cc:260] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } }
14:14:51.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:51.550853 21660 sys_catalog.cc:455] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } } }
14:14:51.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:51.551906 21660 sys_catalog.cc:458] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [sys.catalog]: This master's current role is: FOLLOWER
14:14:51.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:51.552750 21656 sys_catalog.cc:565] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [sys.catalog]: configured and running, proceeding with master startup.
14:14:51.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:51.587334 21671 catalog_manager.cc:1269] Loaded cluster ID: e1b948fcf12a4a879df7c714938a923b
14:14:51.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:51.587780 21671 catalog_manager.cc:1562] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551: loading cluster ID for follower catalog manager: success
14:14:51.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:51.594591 21671 catalog_manager.cc:1584] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551: acquiring CA information for follower catalog manager: success
14:14:51.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 25 14:14:51 dist-test-slave-bg5s krb5kdc[20944](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761401691, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.113.126@KRBTEST.COM for kudu/127.20.113.126@KRBTEST.COM
14:14:51.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:51.600924 21671 catalog_manager.cc:1612] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:14:51.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:51.618201 21585 master_runner.cc:186] Error getting master registration for 127.20.113.125:43287: Network error: Client connection negotiation failed: client connection to 127.20.113.125:43287: connect: Connection refused (error 111),
14:14:51.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:51.645783 21585 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:14:51.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:51.807061 21658 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:14:51.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:51.807766 21658 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:14:51.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:51.808739 21658 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:14:51.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:51.843818 21658 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:14:51.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:51.844065 21658 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:14:51.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:51.844424 21658 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:14:51.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:51.844635 21658 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:14:51.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:51.844816 21658 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:14:51.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:51.845000 21658 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:14:51.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:51.845139 21658 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33059/jwks.json
14:14:51.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:51.883090 21658 master_runner.cc:386] Master server non-default flags:
14:14:51.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:51.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.113.84:43553
14:14:51.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:51.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:14:51.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/data
14:14:51.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/wal
14:14:51.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:14:51.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:14:51.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.113.126:44003,127.20.113.125:43287,127.20.113.124:34873
14:14:51.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kudu.keytab
14:14:51.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.113.125
14:14:51.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:51.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:51.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:14:51.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.113.125:43287
14:14:51.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:14:51.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:14:51.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:14:51.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33059/jwks.json
14:14:51.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:14:51.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:14:51.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:14:51.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:51.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:51.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/data/info.pb
14:14:51.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.113.125
14:14:51.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37077
14:14:51.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:14:51.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:51.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:14:51.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:14:51.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:51.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/logs
14:14:51.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:51.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:14:51.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:14:51.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:14:51.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:14:51.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d0cb013a5aad754f2ed06c25f5ec3d8a1873622
14:14:51.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:14:51.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Oct 2025 13:43:23 UTC on 5fd53c4cbb9d
14:14:51.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8713
14:14:51.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:14:51.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:51.884598 21658 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:14:51.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:51.885769 21658 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:14:51.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:51.896580 21680 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:52.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:52.993783 21690 raft_consensus.cc:493] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:14:52.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:52.995014 21690 raft_consensus.cc:515] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } }
14:14:53.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:53.007329 21607 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.113.125:43287: connect: Connection refused (error 111)
14:14:53.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:53.016167 21690 leader_election.cc:290] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 70c2c7e4f63b4d3f94198427af0a5a3e (127.20.113.125:43287), 661b46d073c04359b178727bdbe67c4d (127.20.113.124:34873)
14:14:53.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:53.029536 21607 leader_election.cc:336] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 70c2c7e4f63b4d3f94198427af0a5a3e (127.20.113.125:43287): Network error: Client connection negotiation failed: client connection to 127.20.113.125:43287: connect: Connection refused (error 111)
14:14:53.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:53.032231 21605 leader_election.cc:336] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 661b46d073c04359b178727bdbe67c4d (127.20.113.124:34873): Network error: Client connection negotiation failed: client connection to 127.20.113.124:34873: connect: Connection refused (error 111)
14:14:53.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:53.032672 21605 leader_election.cc:304] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: f7fa589d2c08468aa6cd09db9c033551; no voters: 661b46d073c04359b178727bdbe67c4d, 70c2c7e4f63b4d3f94198427af0a5a3e
14:14:53.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:53.033411 21690 raft_consensus.cc:2749] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
14:14:53.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:51.898108 21681 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:53.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:53.422077 21682 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1523 milliseconds
14:14:53.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:53.422736 21683 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:53.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:53.422879 21658 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:14:53.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:53.427102 21658 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:14:53.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:53.435252 21658 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:14:53.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:53.436678 21658 hybrid_clock.cc:648] HybridClock initialized: now 1761401693436648 us; error 45 us; skew 500 ppm
14:14:53.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:53.437523 21658 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:14:53.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 25 14:14:53 dist-test-slave-bg5s krb5kdc[20944](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761401693, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.113.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:14:53.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:53.443526 21658 init.cc:377] Logged in from keytab as kudu/127.20.113.125@KRBTEST.COM (short username kudu)
14:14:53.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:53.449349 21658 webserver.cc:492] Webserver started at http://127.20.113.125:37077/ using document root <none> and password file <none>
14:14:53.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:53.450569 21658 fs_manager.cc:362] Metadata directory not provided
14:14:53.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:53.451016 21658 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:14:53.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:53.467624 21658 fs_manager.cc:714] Time spent opening directory manager: real 0.012s user 0.007s sys 0.004s
14:14:53.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:53.474748 21701 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:14:53.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:53.476060 21658 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.005s sys 0.000s
14:14:53.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:53.476402 21658 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/data,/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/wal
14:14:53.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "70c2c7e4f63b4d3f94198427af0a5a3e"
14:14:53.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 14:14:33 on dist-test-slave-bg5s"
14:14:53.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:53.478286 21658 fs_report.cc:389] FS layout report
14:14:53.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:14:53.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/wal
14:14:53.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/wal
14:14:53.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/data/data
14:14:53.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:14:53.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:14:53.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:14:53.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:14:53.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:14:53.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:14:53.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:14:53.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:14:53.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:14:53.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:14:53.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:14:53.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:53.556668 21658 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:14:53.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:53.563730 21658 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:14:53.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:53.564400 21658 kserver.cc:163] Server-wide thread pool size limit: 3276
14:14:53.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:53.652073 21658 rpc_server.cc:307] RPC server started. Bound to: 127.20.113.125:43287
14:14:53.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:53.652127 21752 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.113.125:43287 every 8 connection(s)
14:14:53.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:53.655121 21658 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/data/info.pb
14:14:53.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:53.657070 20933 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu as pid 21658
14:14:53.657 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:14:53.657 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.20.113.124:34873
14:14:53.657 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
14:14:53.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:53.659363 20933 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu
14:14:53.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu
14:14:53.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/wal
14:14:53.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/data
14:14:53.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:14:53.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:14:53.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:14:53.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:53.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:53.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:53.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/logs
14:14:53.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/data/info.pb
14:14:53.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:53.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:14:53.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:14:53.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:14:53.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:14:53.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:53.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:53.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:53.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:14:53.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:14:53.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:14:53.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:14:53.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.113.124:34873
14:14:53.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.113.124
14:14:53.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40487
14:14:53.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kudu.keytab
14:14:53.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.113.124
14:14:53.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:14:53.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:14:53.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:14:53.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:14:53.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.113.84:43553
14:14:53.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:53.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:53.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:14:53.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:14:53.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.113.126:44003,127.20.113.125:43287,127.20.113.124:34873
14:14:53.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:14:53.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33059/jwks.json
14:14:53.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:14:53.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:14:53.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:53.672716 21753 sys_catalog.cc:263] Verifying existing consensus state
14:14:53.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:53.677481 21753 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e: Bootstrap starting.
14:14:53.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:53.721812 21753 log.cc:826] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e: Log is configured to *not* fsync() on all Append() calls
14:14:53.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:53.741448 21753 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:14:53.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:53.742404 21753 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e: Bootstrap complete.
14:14:53.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:53.775344 21753 raft_consensus.cc:359] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } }
14:14:53.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:53.779058 21753 raft_consensus.cc:740] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 70c2c7e4f63b4d3f94198427af0a5a3e, State: Initialized, Role: FOLLOWER
14:14:53.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:53.780337 21753 consensus_queue.cc:260] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } }
14:14:53.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:53.796557 21757 sys_catalog.cc:455] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } } }
14:14:53.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:53.797569 21757 sys_catalog.cc:458] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [sys.catalog]: This master's current role is: FOLLOWER
14:14:53.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:53.798873 21753 sys_catalog.cc:565] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [sys.catalog]: configured and running, proceeding with master startup.
14:14:53.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:53.906008 21768 catalog_manager.cc:1269] Loaded cluster ID: e1b948fcf12a4a879df7c714938a923b
14:14:53.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:53.906262 21768 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e: loading cluster ID for follower catalog manager: success
14:14:53.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:53.914081 21768 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e: acquiring CA information for follower catalog manager: success
14:14:53.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:53.919334 21768 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:14:53.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 25 14:14:53 dist-test-slave-bg5s krb5kdc[20944](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761401693, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.113.125@KRBTEST.COM for kudu/127.20.113.126@KRBTEST.COM
14:14:53.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:53.982751 21658 master_runner.cc:186] Error getting master registration for 127.20.113.124:34873: Network error: Client connection negotiation failed: client connection to 127.20.113.124:34873: connect: Connection refused (error 111),
14:14:53.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:53.998513 21658 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:14:54.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:54.111735 21755 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:14:54.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:54.112275 21755 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:14:54.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:54.112826 21755 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:14:54.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:54.143018 21755 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:14:54.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:54.143285 21755 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:14:54.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:54.143728 21755 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:14:54.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:54.144007 21755 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:14:54.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:54.144304 21755 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:14:54.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:54.144635 21755 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:14:54.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:54.144850 21755 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33059/jwks.json
14:14:54.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:54.183483 21755 master_runner.cc:386] Master server non-default flags:
14:14:54.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:54.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.113.84:43553
14:14:54.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:54.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:14:54.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/data
14:14:54.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/wal
14:14:54.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:14:54.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:14:54.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.113.126:44003,127.20.113.125:43287,127.20.113.124:34873
14:14:54.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kudu.keytab
14:14:54.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.113.124
14:14:54.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:54.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:54.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:14:54.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.113.124:34873
14:14:54.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:14:54.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:14:54.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:14:54.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33059/jwks.json
14:14:54.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:14:54.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:14:54.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:14:54.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:54.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:54.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/data/info.pb
14:14:54.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.113.124
14:14:54.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40487
14:14:54.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:14:54.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:54.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:14:54.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:14:54.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:54.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/logs
14:14:54.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:54.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:14:54.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:14:54.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:14:54.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:14:54.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d0cb013a5aad754f2ed06c25f5ec3d8a1873622
14:14:54.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:14:54.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Oct 2025 13:43:23 UTC on 5fd53c4cbb9d
14:14:54.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8713
14:14:54.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:14:54.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:54.185046 21755 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:14:54.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:54.186054 21755 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:14:54.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:54.197203 21779 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:55.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.083834 21786 raft_consensus.cc:493] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:14:55.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.084594 21786 raft_consensus.cc:515] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } }
14:14:55.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.106510 21786 leader_election.cc:290] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 70c2c7e4f63b4d3f94198427af0a5a3e (127.20.113.125:43287), 661b46d073c04359b178727bdbe67c4d (127.20.113.124:34873)
14:14:55.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:55.130419 21605 leader_election.cc:336] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 661b46d073c04359b178727bdbe67c4d (127.20.113.124:34873): Network error: Client connection negotiation failed: client connection to 127.20.113.124:34873: connect: Connection refused (error 111)
14:14:55.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.139868 21728 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f7fa589d2c08468aa6cd09db9c033551" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" is_pre_election: true
14:14:55.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.141206 21728 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f7fa589d2c08468aa6cd09db9c033551 in term 2.
14:14:55.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.143026 21607 leader_election.cc:304] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 70c2c7e4f63b4d3f94198427af0a5a3e, f7fa589d2c08468aa6cd09db9c033551; no voters: 661b46d073c04359b178727bdbe67c4d
14:14:55.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.144767 21786 raft_consensus.cc:2804] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [term 2 FOLLOWER]: Leader pre-election won for term 3
14:14:55.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.145229 21786 raft_consensus.cc:493] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:14:55.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.145696 21786 raft_consensus.cc:3060] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [term 2 FOLLOWER]: Advancing to term 3
14:14:55.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.161051 21786 raft_consensus.cc:515] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } }
14:14:55.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.165017 21728 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f7fa589d2c08468aa6cd09db9c033551" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e"
14:14:55.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.165735 21728 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [term 2 FOLLOWER]: Advancing to term 3
14:14:55.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:55.185982 21605 leader_election.cc:336] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 661b46d073c04359b178727bdbe67c4d (127.20.113.124:34873): Network error: Client connection negotiation failed: client connection to 127.20.113.124:34873: connect: Connection refused (error 111)
14:14:55.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.183420 21786 leader_election.cc:290] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [CANDIDATE]: Term 3 election: Requested vote from peers 70c2c7e4f63b4d3f94198427af0a5a3e (127.20.113.125:43287), 661b46d073c04359b178727bdbe67c4d (127.20.113.124:34873)
14:14:55.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.179312 21728 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f7fa589d2c08468aa6cd09db9c033551 in term 3.
14:14:55.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.189240 21607 leader_election.cc:304] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 70c2c7e4f63b4d3f94198427af0a5a3e, f7fa589d2c08468aa6cd09db9c033551; no voters: 661b46d073c04359b178727bdbe67c4d
14:14:55.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.190752 21786 raft_consensus.cc:2804] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [term 3 FOLLOWER]: Leader election won for term 3
14:14:55.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.199569 21786 raft_consensus.cc:697] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [term 3 LEADER]: Becoming Leader. State: Replica: f7fa589d2c08468aa6cd09db9c033551, State: Running, Role: LEADER
14:14:55.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.201543 21786 consensus_queue.cc:237] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } }
14:14:55.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.225327 21792 sys_catalog.cc:455] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f7fa589d2c08468aa6cd09db9c033551. Latest consensus state: current_term: 3 leader_uuid: "f7fa589d2c08468aa6cd09db9c033551" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } } }
14:14:55.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.226933 21792 sys_catalog.cc:458] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [sys.catalog]: This master's current role is: LEADER
14:14:55.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.233551 21794 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
14:14:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.256508 21794 catalog_manager.cc:1494] Initializing Kudu cluster ID...
14:14:55.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.268317 21794 catalog_manager.cc:1269] Loaded cluster ID: e1b948fcf12a4a879df7c714938a923b
14:14:55.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.279563 21794 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
14:14:55.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.286399 21794 catalog_manager.cc:1514] Loading token signing keys...
14:14:55.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.292275 21794 catalog_manager.cc:6033] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551: Loaded TSK: 0
14:14:55.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.294416 21794 catalog_manager.cc:1524] Initializing in-progress tserver states...
14:14:55.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:55.597290 21605 consensus_peers.cc:597] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 -> Peer 661b46d073c04359b178727bdbe67c4d (127.20.113.124:34873): Couldn't send request to peer 661b46d073c04359b178727bdbe67c4d. Status: Network error: Client connection negotiation failed: client connection to 127.20.113.124:34873: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:14:55.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.802143 21728 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [term 3 FOLLOWER]: Refusing update from remote peer f7fa589d2c08468aa6cd09db9c033551: Log matching property violated. Preceding OpId in replica: term: 2 index: 5. Preceding OpId from leader: term: 3 index: 6. (index mismatch)
14:14:55.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.803942 21792 consensus_queue.cc:1048] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [LEADER]: Connected to new peer: Peer: permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
14:14:55.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:54.198812 21780 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:55.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:55.837795 21782 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:55.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:55.840816 21781 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1642 milliseconds
14:14:55.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.841387 21755 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:14:55.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.842864 21755 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:14:55.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.852988 21755 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:14:55.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.857971 21755 hybrid_clock.cc:648] HybridClock initialized: now 1761401695857536 us; error 420 us; skew 500 ppm
14:14:55.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.859031 21755 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:14:55.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.862974 21796 sys_catalog.cc:455] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [sys.catalog]: SysCatalogTable state changed. Reason: New leader f7fa589d2c08468aa6cd09db9c033551. Latest consensus state: current_term: 3 leader_uuid: "f7fa589d2c08468aa6cd09db9c033551" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } } }
14:14:55.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.863713 21796 sys_catalog.cc:458] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [sys.catalog]: This master's current role is: FOLLOWER
14:14:55.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 25 14:14:55 dist-test-slave-bg5s krb5kdc[20944](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761401695, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.113.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:14:55.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.866751 21755 init.cc:377] Logged in from keytab as kudu/127.20.113.124@KRBTEST.COM (short username kudu)
14:14:55.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.873905 21799 sys_catalog.cc:455] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "f7fa589d2c08468aa6cd09db9c033551" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } } }
14:14:55.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.875458 21799 sys_catalog.cc:458] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [sys.catalog]: This master's current role is: LEADER
14:14:55.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.882778 21796 sys_catalog.cc:455] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "f7fa589d2c08468aa6cd09db9c033551" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } } }
14:14:55.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.883450 21796 sys_catalog.cc:458] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [sys.catalog]: This master's current role is: FOLLOWER
14:14:55.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.884474 21755 webserver.cc:492] Webserver started at http://127.20.113.124:40487/ using document root <none> and password file <none>
14:14:55.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.885350 21755 fs_manager.cc:362] Metadata directory not provided
14:14:55.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.885550 21755 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:14:55.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.893187 21755 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.000s sys 0.005s
14:14:55.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.897876 21807 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:14:55.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.898983 21755 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
14:14:55.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.899296 21755 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/data,/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/wal
14:14:55.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "661b46d073c04359b178727bdbe67c4d"
14:14:55.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 14:14:35 on dist-test-slave-bg5s"
14:14:55.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.901408 21755 fs_report.cc:389] FS layout report
14:14:55.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:14:55.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/wal
14:14:55.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/wal
14:14:55.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/data/data
14:14:55.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:14:55.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:14:55.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:14:55.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:14:55.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:14:55.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:14:55.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:14:55.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:14:55.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:14:55.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:14:55.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:14:55.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.967947 21755 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:14:55.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.973227 21755 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:14:55.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:55.973634 21755 kserver.cc:163] Server-wide thread pool size limit: 3276
14:14:56.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:56.058203 21755 rpc_server.cc:307] RPC server started. Bound to: 127.20.113.124:34873
14:14:56.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:56.058259 21858 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.113.124:34873 every 8 connection(s)
14:14:56.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:56.061003 21755 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/data/info.pb
14:14:56.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:56.066525 20933 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu as pid 21755
14:14:56.067 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:14:56.067 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.20.113.126:44003(127.20.113.126:44003)] connecting to peer
14:14:56.071 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.20.113.126:44003(127.20.113.126:44003)] Successfully connected to peer
14:14:56.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:56.073716 21859 sys_catalog.cc:263] Verifying existing consensus state
14:14:56.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:56.078996 21859 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d: Bootstrap starting.
14:14:56.099 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x37f0eb61, L:/127.0.0.1:35610 - R:/127.20.113.126:44003] using TOKEN/null
14:14:56.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:56.127568 21859 log.cc:826] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d: Log is configured to *not* fsync() on all Append() calls
14:14:56.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 25 14:14:56 dist-test-slave-bg5s krb5kdc[20944](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761401691, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.113.126@KRBTEST.COM for kudu/127.20.113.124@KRBTEST.COM
14:14:56.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:56.147996 21859 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:14:56.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:56.148908 21859 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d: Bootstrap complete.
14:14:56.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:56.177284 21859 raft_consensus.cc:359] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } }
14:14:56.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:56.180354 21859 raft_consensus.cc:740] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 661b46d073c04359b178727bdbe67c4d, State: Initialized, Role: FOLLOWER
14:14:56.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:56.181273 21859 consensus_queue.cc:260] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } }
14:14:56.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:56.183972 21865 sys_catalog.cc:455] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } } }
14:14:56.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:56.184674 21865 sys_catalog.cc:458] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [sys.catalog]: This master's current role is: FOLLOWER
14:14:56.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:56.185729 21859 sys_catalog.cc:565] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [sys.catalog]: configured and running, proceeding with master startup.
14:14:56.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:56.186261 21834 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [term 2 FOLLOWER]: Advancing to term 3
14:14:56.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:56.205991 21875 sys_catalog.cc:455] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "f7fa589d2c08468aa6cd09db9c033551" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } } }
14:14:56.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:56.206774 21875 sys_catalog.cc:458] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [sys.catalog]: This master's current role is: FOLLOWER
14:14:56.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:56.212015 21865 sys_catalog.cc:455] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [sys.catalog]: SysCatalogTable state changed. Reason: New leader f7fa589d2c08468aa6cd09db9c033551. Latest consensus state: current_term: 3 leader_uuid: "f7fa589d2c08468aa6cd09db9c033551" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } } }
14:14:56.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:56.212864 21865 sys_catalog.cc:458] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [sys.catalog]: This master's current role is: FOLLOWER
14:14:56.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:56.226615 21879 catalog_manager.cc:1269] Loaded cluster ID: e1b948fcf12a4a879df7c714938a923b
14:14:56.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:56.227865 21879 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d: loading cluster ID for follower catalog manager: success
14:14:56.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 25 14:14:56 dist-test-slave-bg5s krb5kdc[20944](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761401695, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.113.124@KRBTEST.COM for kudu/127.20.113.126@KRBTEST.COM
14:14:56.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:56.237917 21879 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d: acquiring CA information for follower catalog manager: success
14:14:56.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:56.242770 21879 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:14:56.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:56.541561 21799 sys_catalog.cc:455] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "f7fa589d2c08468aa6cd09db9c033551" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } } }
14:14:56.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:56.542239 21799 sys_catalog.cc:458] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [sys.catalog]: This master's current role is: LEADER
14:14:59.154 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.20.113.126:44003
14:14:59.154 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
14:14:59.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:59.154771 20933 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu with pid 21585
14:14:59.183 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.20.113.126:44003(127.20.113.126:44003)] handling channelInactive
14:14:59.183 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.113.126:44003(127.20.113.126:44003)] cleaning up while in state READY due to: connection closed
14:14:59.183 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:14:59.183 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.20.113.125:43287
14:14:59.184 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
14:14:59.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:59.184379 20933 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu with pid 21658
14:14:59.210 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:14:59.211 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.20.113.124:34873
14:14:59.211 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
14:14:59.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:59.211555 20933 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu with pid 21755
14:14:59.238 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:14:59.238 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.20.113.126:44003
14:14:59.238 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
14:14:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:59.239759 20933 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu
14:14:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu
14:14:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/wal
14:14:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/data
14:14:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:14:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:14:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:14:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/logs
14:14:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/data/info.pb
14:14:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:14:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:14:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:14:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:14:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:14:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:14:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:14:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:14:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.113.126:44003
14:14:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.113.126
14:14:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45853
14:14:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kudu.keytab
14:14:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.113.126
14:14:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:14:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:14:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:14:59.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:14:59.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.113.84:43553
14:14:59.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:59.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:59.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:14:59.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:14:59.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.113.126:44003,127.20.113.125:43287,127.20.113.124:34873
14:14:59.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:14:59.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33059/jwks.json
14:14:59.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:14:59.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:14:59.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:59.549847 21890 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:14:59.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:59.550371 21890 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:14:59.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:59.551085 21890 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:14:59.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:59.581789 21890 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:14:59.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:59.582118 21890 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:14:59.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:59.582540 21890 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:14:59.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:59.582813 21890 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:14:59.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:59.583072 21890 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:14:59.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:59.583348 21890 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:14:59.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:59.583618 21890 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33059/jwks.json
14:14:59.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:59.621057 21890 master_runner.cc:386] Master server non-default flags:
14:14:59.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:59.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.113.84:43553
14:14:59.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:59.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:14:59.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/data
14:14:59.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/wal
14:14:59.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:14:59.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:14:59.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.113.126:44003,127.20.113.125:43287,127.20.113.124:34873
14:14:59.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kudu.keytab
14:14:59.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.113.126
14:14:59.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:59.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:59.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:14:59.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.113.126:44003
14:14:59.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:14:59.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:14:59.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:14:59.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33059/jwks.json
14:14:59.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:14:59.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:14:59.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:14:59.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:59.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:59.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/data/info.pb
14:14:59.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.113.126
14:14:59.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45853
14:14:59.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:14:59.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:59.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:14:59.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:14:59.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:59.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/logs
14:14:59.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:59.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:14:59.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:14:59.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:14:59.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:14:59.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d0cb013a5aad754f2ed06c25f5ec3d8a1873622
14:14:59.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:14:59.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Oct 2025 13:43:23 UTC on 5fd53c4cbb9d
14:14:59.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8713
14:14:59.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:14:59.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:59.622650 21890 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:14:59.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:14:59.623761 21890 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:14:59.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:59.633567 21896 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:15:00.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:14:59.635537 21897 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:15:00.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:15:00.914263 21899 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:15:00.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:15:00.916863 21898 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1277 milliseconds
14:15:00.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:00.917013 21890 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:15:00.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:00.918292 21890 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:15:00.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:00.920799 21890 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:15:00.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:00.922215 21890 hybrid_clock.cc:648] HybridClock initialized: now 1761401700922147 us; error 45 us; skew 500 ppm
14:15:00.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:00.923076 21890 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:15:00.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 25 14:15:00 dist-test-slave-bg5s krb5kdc[20944](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761401700, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.113.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:15:00.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:00.928658 21890 init.cc:377] Logged in from keytab as kudu/127.20.113.126@KRBTEST.COM (short username kudu)
14:15:00.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:00.933866 21890 webserver.cc:492] Webserver started at http://127.20.113.126:45853/ using document root <none> and password file <none>
14:15:00.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:00.934747 21890 fs_manager.cc:362] Metadata directory not provided
14:15:00.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:00.934969 21890 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:15:00.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:00.942616 21890 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.006s sys 0.000s
14:15:00.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:00.946913 21909 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:15:00.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:00.947923 21890 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
14:15:00.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:00.948220 21890 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/data,/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/wal
14:15:00.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f7fa589d2c08468aa6cd09db9c033551"
14:15:00.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 14:14:31 on dist-test-slave-bg5s"
14:15:00.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:00.950114 21890 fs_report.cc:389] FS layout report
14:15:00.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:15:00.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/wal
14:15:00.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/wal
14:15:00.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/data/data
14:15:00.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:15:00.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:15:00.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:15:00.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:15:00.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:15:00.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:15:00.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:15:00.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:15:00.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:15:00.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:15:00.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:15:00.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:00.997421 21890 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:15:01.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:01.002844 21890 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:15:01.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:01.003288 21890 kserver.cc:163] Server-wide thread pool size limit: 3276
14:15:01.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:01.072414 21890 rpc_server.cc:307] RPC server started. Bound to: 127.20.113.126:44003
14:15:01.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:01.072500 21960 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.113.126:44003 every 8 connection(s)
14:15:01.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:01.075021 21890 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-0/data/info.pb
14:15:01.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:01.082572 20933 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu as pid 21890
14:15:01.083 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:15:01.083 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.20.113.125:43287
14:15:01.083 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
14:15:01.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:01.085114 20933 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu
14:15:01.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu
14:15:01.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/wal
14:15:01.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/data
14:15:01.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:15:01.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:15:01.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:15:01.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:15:01.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:15:01.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:15:01.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/logs
14:15:01.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/data/info.pb
14:15:01.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:15:01.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:15:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:15:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:15:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:15:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:15:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:15:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:15:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:15:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:15:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:15:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:15:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.113.125:43287
14:15:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.113.125
14:15:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37077
14:15:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kudu.keytab
14:15:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.113.125
14:15:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:15:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:15:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:15:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:15:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.113.84:43553
14:15:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:15:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:15:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:15:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:15:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.113.126:44003,127.20.113.125:43287,127.20.113.124:34873
14:15:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:15:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33059/jwks.json
14:15:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:15:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:15:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:01.086678 21961 sys_catalog.cc:263] Verifying existing consensus state
14:15:01.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:01.094632 21961 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551: Bootstrap starting.
14:15:01.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:01.149020 21961 log.cc:826] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551: Log is configured to *not* fsync() on all Append() calls
14:15:01.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:01.170607 21961 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:15:01.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:01.171700 21961 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551: Bootstrap complete.
14:15:01.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:01.208609 21961 raft_consensus.cc:359] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } }
14:15:01.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:01.212530 21961 raft_consensus.cc:740] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: f7fa589d2c08468aa6cd09db9c033551, State: Initialized, Role: FOLLOWER
14:15:01.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:01.213601 21961 consensus_queue.cc:260] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } }
14:15:01.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:01.216377 21965 sys_catalog.cc:455] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } } }
14:15:01.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:01.216955 21965 sys_catalog.cc:458] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [sys.catalog]: This master's current role is: FOLLOWER
14:15:01.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:01.218794 21961 sys_catalog.cc:565] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [sys.catalog]: configured and running, proceeding with master startup.
14:15:01.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:01.263521 21976 catalog_manager.cc:1269] Loaded cluster ID: e1b948fcf12a4a879df7c714938a923b
14:15:01.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:01.263968 21976 catalog_manager.cc:1562] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551: loading cluster ID for follower catalog manager: success
14:15:01.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:01.272967 21976 catalog_manager.cc:1584] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551: acquiring CA information for follower catalog manager: success
14:15:01.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 25 14:15:01 dist-test-slave-bg5s krb5kdc[20944](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761401700, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.113.126@KRBTEST.COM for kudu/127.20.113.126@KRBTEST.COM
14:15:01.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:01.277880 21976 catalog_manager.cc:1612] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:15:01.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:01.298417 21890 master_runner.cc:186] Error getting master registration for 127.20.113.125:43287: Network error: Client connection negotiation failed: client connection to 127.20.113.125:43287: connect: Connection refused (error 111),
14:15:01.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:01.324111 21890 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:15:01.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:15:01.459070 21963 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:15:01.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:15:01.459676 21963 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:15:01.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:15:01.460348 21963 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:15:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:15:01.490811 21963 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:15:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:15:01.491138 21963 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:15:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:15:01.491575 21963 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:15:01.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:15:01.491887 21963 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:15:01.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:15:01.492203 21963 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:15:01.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:15:01.492496 21963 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:15:01.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:15:01.492733 21963 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33059/jwks.json
14:15:01.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:01.543511 21963 master_runner.cc:386] Master server non-default flags:
14:15:01.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:15:01.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.113.84:43553
14:15:01.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:15:01.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:15:01.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/data
14:15:01.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/wal
14:15:01.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:15:01.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:15:01.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.113.126:44003,127.20.113.125:43287,127.20.113.124:34873
14:15:01.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kudu.keytab
14:15:01.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.113.125
14:15:01.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:15:01.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:15:01.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:15:01.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.113.125:43287
14:15:01.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:15:01.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:15:01.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:15:01.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33059/jwks.json
14:15:01.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:15:01.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:15:01.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:15:01.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:15:01.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:15:01.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/data/info.pb
14:15:01.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.113.125
14:15:01.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37077
14:15:01.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:15:01.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:15:01.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:15:01.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:15:01.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:15:01.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/logs
14:15:01.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:15:01.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:15:01.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:15:01.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:15:01.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:15:01.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d0cb013a5aad754f2ed06c25f5ec3d8a1873622
14:15:01.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:15:01.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Oct 2025 13:43:23 UTC on 5fd53c4cbb9d
14:15:01.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8713
14:15:01.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:15:01.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:01.545127 21963 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:15:01.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:01.546424 21963 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:15:01.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:15:01.559082 21985 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:15:02.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:02.379022 21991 raft_consensus.cc:493] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:15:02.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:02.380061 21991 raft_consensus.cc:515] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } }
14:15:02.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:02.410053 21991 leader_election.cc:290] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 70c2c7e4f63b4d3f94198427af0a5a3e (127.20.113.125:43287), 661b46d073c04359b178727bdbe67c4d (127.20.113.124:34873)
14:15:02.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:15:02.414021 21913 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.113.125:43287: connect: Connection refused (error 111)
14:15:02.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:15:02.427291 21913 leader_election.cc:336] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 70c2c7e4f63b4d3f94198427af0a5a3e (127.20.113.125:43287): Network error: Client connection negotiation failed: client connection to 127.20.113.125:43287: connect: Connection refused (error 111)
14:15:02.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:15:02.430130 21911 leader_election.cc:336] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 661b46d073c04359b178727bdbe67c4d (127.20.113.124:34873): Network error: Client connection negotiation failed: client connection to 127.20.113.124:34873: connect: Connection refused (error 111)
14:15:02.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:02.430681 21911 leader_election.cc:304] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: f7fa589d2c08468aa6cd09db9c033551; no voters: 661b46d073c04359b178727bdbe67c4d, 70c2c7e4f63b4d3f94198427af0a5a3e
14:15:02.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:02.431674 21991 raft_consensus.cc:2749] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
14:15:03.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:15:03.352201 21963 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.791s user 0.500s sys 0.936s
14:15:03.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:15:03.352845 21963 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.792s user 0.501s sys 0.937s
14:15:03.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:15:02.960652 21984 debug-util.cc:398] Leaking SignalData structure 0x7b0800027080 after lost signal to thread 21963
14:15:03.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:15:01.560652 21986 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:15:03.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:15:03.355096 21988 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:15:03.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:15:03.358831 21987 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1798 milliseconds
14:15:03.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:03.358871 21963 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:15:03.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:03.360843 21963 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:15:03.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:03.363826 21963 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:15:03.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:03.365238 21963 hybrid_clock.cc:648] HybridClock initialized: now 1761401703365167 us; error 69 us; skew 500 ppm
14:15:03.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:03.366247 21963 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:15:03.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 25 14:15:03 dist-test-slave-bg5s krb5kdc[20944](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761401703, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.113.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:15:03.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:03.373602 21963 init.cc:377] Logged in from keytab as kudu/127.20.113.125@KRBTEST.COM (short username kudu)
14:15:03.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:03.379058 21963 webserver.cc:492] Webserver started at http://127.20.113.125:37077/ using document root <none> and password file <none>
14:15:03.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:03.380306 21963 fs_manager.cc:362] Metadata directory not provided
14:15:03.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:03.380553 21963 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:15:03.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:03.390656 21963 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.005s sys 0.001s
14:15:03.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:03.396104 22002 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:15:03.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:03.397168 21963 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
14:15:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:03.397431 21963 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/data,/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/wal
14:15:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "70c2c7e4f63b4d3f94198427af0a5a3e"
14:15:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 14:14:33 on dist-test-slave-bg5s"
14:15:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:03.399266 21963 fs_report.cc:389] FS layout report
14:15:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:15:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/wal
14:15:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/wal
14:15:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/data/data
14:15:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:15:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:15:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:15:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:15:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:15:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:15:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:15:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:15:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:15:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:15:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:15:03.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:03.449442 21963 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:15:03.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:03.454980 21963 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:15:03.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:03.455444 21963 kserver.cc:163] Server-wide thread pool size limit: 3276
14:15:03.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:03.561374 21963 rpc_server.cc:307] RPC server started. Bound to: 127.20.113.125:43287
14:15:03.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:03.561443 22055 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.113.125:43287 every 8 connection(s)
14:15:03.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:03.564299 21963 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-1/data/info.pb
14:15:03.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:03.571002 20933 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu as pid 21963
14:15:03.571 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:15:03.571 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.20.113.124:34873
14:15:03.571 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
14:15:03.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:03.573596 20933 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu
14:15:03.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu
14:15:03.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/wal
14:15:03.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/data
14:15:03.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:15:03.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:15:03.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:15:03.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:15:03.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:15:03.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:15:03.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/logs
14:15:03.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/data/info.pb
14:15:03.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:15:03.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:15:03.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:15:03.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:15:03.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:15:03.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:15:03.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:15:03.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:15:03.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:15:03.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:15:03.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:15:03.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:15:03.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.113.124:34873
14:15:03.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.113.124
14:15:03.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40487
14:15:03.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kudu.keytab
14:15:03.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.113.124
14:15:03.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:15:03.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:15:03.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:15:03.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:15:03.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.113.84:43553
14:15:03.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:15:03.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:15:03.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:15:03.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:15:03.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.113.126:44003,127.20.113.125:43287,127.20.113.124:34873
14:15:03.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:15:03.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33059/jwks.json
14:15:03.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:15:03.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:15:03.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:03.577445 22056 sys_catalog.cc:263] Verifying existing consensus state
14:15:03.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:03.583866 22056 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e: Bootstrap starting.
14:15:03.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:03.636353 22056 log.cc:826] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e: Log is configured to *not* fsync() on all Append() calls
14:15:03.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:03.657730 22056 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:15:03.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:03.658640 22056 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e: Bootstrap complete.
14:15:03.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:03.690938 22056 raft_consensus.cc:359] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } }
14:15:03.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:03.694658 22056 raft_consensus.cc:740] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 70c2c7e4f63b4d3f94198427af0a5a3e, State: Initialized, Role: FOLLOWER
14:15:03.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:03.695789 22056 consensus_queue.cc:260] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } }
14:15:03.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:03.699071 22060 sys_catalog.cc:455] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } } }
14:15:03.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:03.700234 22060 sys_catalog.cc:458] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [sys.catalog]: This master's current role is: FOLLOWER
14:15:03.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:03.701032 22056 sys_catalog.cc:565] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [sys.catalog]: configured and running, proceeding with master startup.
14:15:03.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:03.767129 22071 catalog_manager.cc:1269] Loaded cluster ID: e1b948fcf12a4a879df7c714938a923b
14:15:03.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:03.767618 22071 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e: loading cluster ID for follower catalog manager: success
14:15:03.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:03.781988 22071 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e: acquiring CA information for follower catalog manager: success
14:15:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:03.795130 22071 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:15:03.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 25 14:15:03 dist-test-slave-bg5s krb5kdc[20944](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761401703, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.113.125@KRBTEST.COM for kudu/127.20.113.126@KRBTEST.COM
14:15:03.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:03.866988 21963 master_runner.cc:186] Error getting master registration for 127.20.113.124:34873: Network error: Client connection negotiation failed: client connection to 127.20.113.124:34873: connect: Connection refused (error 111),
14:15:03.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:03.898603 21963 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:15:03.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:15:03.993345 22058 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:15:03.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:15:03.994086 22058 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:15:03.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:15:03.995182 22058 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:15:04.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:15:04.053035 22058 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:15:04.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:15:04.053369 22058 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:15:04.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:15:04.053782 22058 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:15:04.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:15:04.054095 22058 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:15:04.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:15:04.054406 22058 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:15:04.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:15:04.054710 22058 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:15:04.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:15:04.054934 22058 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33059/jwks.json
14:15:04.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:04.121003 22058 master_runner.cc:386] Master server non-default flags:
14:15:04.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:15:04.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.113.84:43553
14:15:04.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:15:04.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:15:04.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/data
14:15:04.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/wal
14:15:04.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:15:04.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:15:04.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.113.126:44003,127.20.113.125:43287,127.20.113.124:34873
14:15:04.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/krb5kdc/kudu.keytab
14:15:04.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.113.124
14:15:04.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:15:04.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:15:04.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:15:04.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.113.124:34873
14:15:04.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:15:04.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:15:04.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:15:04.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33059/jwks.json
14:15:04.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:15:04.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:15:04.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:15:04.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:15:04.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:15:04.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/data/info.pb
14:15:04.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.113.124
14:15:04.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40487
14:15:04.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:15:04.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:15:04.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:15:04.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:15:04.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:15:04.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/logs
14:15:04.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:15:04.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:15:04.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:15:04.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:15:04.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:15:04.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d0cb013a5aad754f2ed06c25f5ec3d8a1873622
14:15:04.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:15:04.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Oct 2025 13:43:23 UTC on 5fd53c4cbb9d
14:15:04.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8713
14:15:04.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:15:04.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:04.122713 22058 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:15:04.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:04.123960 22058 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:15:04.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:15:04.136680 22081 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:15:04.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:04.661042 22086 raft_consensus.cc:493] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:15:04.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:04.661921 22086 raft_consensus.cc:515] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } }
14:15:04.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:04.664034 22086 leader_election.cc:290] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 70c2c7e4f63b4d3f94198427af0a5a3e (127.20.113.125:43287), 661b46d073c04359b178727bdbe67c4d (127.20.113.124:34873)
14:15:04.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:04.728924 22030 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f7fa589d2c08468aa6cd09db9c033551" candidate_term: 4 candidate_status { last_received { term: 3 index: 6 } } ignore_live_leader: false dest_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" is_pre_election: true
14:15:04.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:04.730065 22030 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f7fa589d2c08468aa6cd09db9c033551 in term 3.
14:15:04.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:04.731595 21913 leader_election.cc:304] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 70c2c7e4f63b4d3f94198427af0a5a3e, f7fa589d2c08468aa6cd09db9c033551; no voters:
14:15:04.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:04.732964 22086 raft_consensus.cc:2804] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [term 3 FOLLOWER]: Leader pre-election won for term 4
14:15:04.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:04.733450 22086 raft_consensus.cc:493] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [term 3 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:15:04.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:04.733979 22086 raft_consensus.cc:3060] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [term 3 FOLLOWER]: Advancing to term 4
14:15:04.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:15:04.735029 21911 leader_election.cc:336] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 661b46d073c04359b178727bdbe67c4d (127.20.113.124:34873): Network error: Client connection negotiation failed: client connection to 127.20.113.124:34873: connect: Connection refused (error 111)
14:15:04.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:04.752044 22086 raft_consensus.cc:515] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } }
14:15:04.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:04.755455 22030 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f7fa589d2c08468aa6cd09db9c033551" candidate_term: 4 candidate_status { last_received { term: 3 index: 6 } } ignore_live_leader: false dest_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e"
14:15:04.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:04.755997 22086 leader_election.cc:290] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [CANDIDATE]: Term 4 election: Requested vote from peers 70c2c7e4f63b4d3f94198427af0a5a3e (127.20.113.125:43287), 661b46d073c04359b178727bdbe67c4d (127.20.113.124:34873)
14:15:04.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:04.756032 22030 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [term 3 FOLLOWER]: Advancing to term 4
14:15:04.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:15:04.759714 21911 leader_election.cc:336] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 661b46d073c04359b178727bdbe67c4d (127.20.113.124:34873): Network error: Client connection negotiation failed: client connection to 127.20.113.124:34873: connect: Connection refused (error 111)
14:15:04.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:04.768195 22030 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f7fa589d2c08468aa6cd09db9c033551 in term 4.
14:15:04.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:04.769440 21913 leader_election.cc:304] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 70c2c7e4f63b4d3f94198427af0a5a3e, f7fa589d2c08468aa6cd09db9c033551; no voters: 661b46d073c04359b178727bdbe67c4d
14:15:04.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:04.770486 22086 raft_consensus.cc:2804] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [term 4 FOLLOWER]: Leader election won for term 4
14:15:04.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:04.808729 22086 raft_consensus.cc:697] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [term 4 LEADER]: Becoming Leader. State: Replica: f7fa589d2c08468aa6cd09db9c033551, State: Running, Role: LEADER
14:15:04.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:04.810140 22086 consensus_queue.cc:237] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } }
14:15:04.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:04.850939 22091 sys_catalog.cc:455] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f7fa589d2c08468aa6cd09db9c033551. Latest consensus state: current_term: 4 leader_uuid: "f7fa589d2c08468aa6cd09db9c033551" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } } }
14:15:04.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:04.852470 22091 sys_catalog.cc:458] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [sys.catalog]: This master's current role is: LEADER
14:15:04.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:04.858685 22093 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
14:15:04.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:04.882892 22093 catalog_manager.cc:1494] Initializing Kudu cluster ID...
14:15:04.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:04.901844 22093 catalog_manager.cc:1269] Loaded cluster ID: e1b948fcf12a4a879df7c714938a923b
14:15:04.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:04.902240 22093 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
14:15:04.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:04.916529 22093 catalog_manager.cc:1514] Loading token signing keys...
14:15:04.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:04.924899 22093 catalog_manager.cc:6033] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551: Loaded TSK: 0
14:15:04.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:04.926635 22093 catalog_manager.cc:1524] Initializing in-progress tserver states...
14:15:05.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:05.245628 22030 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [term 4 FOLLOWER]: Refusing update from remote peer f7fa589d2c08468aa6cd09db9c033551: Log matching property violated. Preceding OpId in replica: term: 3 index: 6. Preceding OpId from leader: term: 4 index: 7. (index mismatch)
14:15:05.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:05.252316 22091 consensus_queue.cc:1048] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [LEADER]: Connected to new peer: Peer: permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
14:15:05.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:05.293135 22096 sys_catalog.cc:455] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [sys.catalog]: SysCatalogTable state changed. Reason: New leader f7fa589d2c08468aa6cd09db9c033551. Latest consensus state: current_term: 4 leader_uuid: "f7fa589d2c08468aa6cd09db9c033551" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } } }
14:15:05.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:05.294109 22096 sys_catalog.cc:458] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [sys.catalog]: This master's current role is: FOLLOWER
14:15:05.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:05.299414 22091 sys_catalog.cc:455] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "f7fa589d2c08468aa6cd09db9c033551" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } } }
14:15:05.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:05.301726 22091 sys_catalog.cc:458] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 [sys.catalog]: This master's current role is: LEADER
14:15:05.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:05.316120 22096 sys_catalog.cc:455] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "f7fa589d2c08468aa6cd09db9c033551" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } } }
14:15:05.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:05.316978 22096 sys_catalog.cc:458] T 00000000000000000000000000000000 P 70c2c7e4f63b4d3f94198427af0a5a3e [sys.catalog]: This master's current role is: FOLLOWER
14:15:05.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:15:05.319329 21911 consensus_peers.cc:597] T 00000000000000000000000000000000 P f7fa589d2c08468aa6cd09db9c033551 -> Peer 661b46d073c04359b178727bdbe67c4d (127.20.113.124:34873): Couldn't send request to peer 661b46d073c04359b178727bdbe67c4d. Status: Network error: Client connection negotiation failed: client connection to 127.20.113.124:34873: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:15:05.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:15:05.538285 22080 debug-util.cc:398] Leaking SignalData structure 0x7b0800027080 after lost signal to thread 22058
14:15:05.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:15:05.543743 22058 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.406s user 0.440s sys 0.782s
14:15:05.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:15:05.545424 22058 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.408s user 0.441s sys 0.782s
14:15:05.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:15:04.140136 22082 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:15:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:15:05.575636 22103 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:15:05.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:05.579583 22058 server_base.cc:1047] running on GCE node
14:15:05.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:05.580757 22058 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:15:05.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:05.584059 22058 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:15:05.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:05.585809 22058 hybrid_clock.cc:648] HybridClock initialized: now 1761401705585772 us; error 48 us; skew 500 ppm
14:15:05.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:05.586580 22058 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:15:05.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 25 14:15:05 dist-test-slave-bg5s krb5kdc[20944](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761401705, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.113.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:15:05.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:05.594658 22058 init.cc:377] Logged in from keytab as kudu/127.20.113.124@KRBTEST.COM (short username kudu)
14:15:05.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:05.603411 22058 webserver.cc:492] Webserver started at http://127.20.113.124:40487/ using document root <none> and password file <none>
14:15:05.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:05.604525 22058 fs_manager.cc:362] Metadata directory not provided
14:15:05.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:05.604745 22058 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:15:05.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:05.612965 22058 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.006s sys 0.002s
14:15:05.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:05.617627 22109 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:15:05.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:05.618870 22058 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
14:15:05.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:05.619324 22058 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/data,/tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/wal
14:15:05.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "661b46d073c04359b178727bdbe67c4d"
14:15:05.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 14:14:35 on dist-test-slave-bg5s"
14:15:05.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:05.621904 22058 fs_report.cc:389] FS layout report
14:15:05.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:15:05.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/wal
14:15:05.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/wal
14:15:05.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/data/data
14:15:05.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:15:05.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:15:05.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:15:05.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:15:05.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:15:05.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:15:05.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:15:05.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:15:05.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:15:05.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:15:05.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:15:05.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:05.712580 22058 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:15:05.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:05.723701 22058 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:15:05.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:05.724160 22058 kserver.cc:163] Server-wide thread pool size limit: 3276
14:15:05.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:05.836028 22058 rpc_server.cc:307] RPC server started. Bound to: 127.20.113.124:34873
14:15:05.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:05.836118 22163 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.113.124:34873 every 8 connection(s)
14:15:05.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:05.839866 22058 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskB9qii1/test-tmp/mini-kudu-cluster767459152376820762/master-2/data/info.pb
14:15:05.842 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:15:05.843 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.20.113.126:44003(127.20.113.126:44003)] connecting to peer
14:15:05.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:05.842308 20933 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu as pid 22058
14:15:05.847 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.20.113.126:44003(127.20.113.126:44003)] Successfully connected to peer
14:15:05.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:05.883685 22164 sys_catalog.cc:263] Verifying existing consensus state
14:15:05.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 25 14:15:05 dist-test-slave-bg5s krb5kdc[20944](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761401700, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.113.126@KRBTEST.COM for kudu/127.20.113.124@KRBTEST.COM
14:15:05.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:05.891139 22164 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d: Bootstrap starting.
14:15:05.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:15:05.908187 22166 negotiation.cc:344] Unauthorized connection attempt: Server connection negotiation failed: server connection from 127.0.0.1:60878: token expired
14:15:05.913 [DEBUG - kudu-nio-0] (Negotiator.java:352) peer /127.20.113.126:44003 sent connection negotiation error: Not authorized: token expired
14:15:05.915 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.20.113.126:44003(127.20.113.126:44003)] handling channelInactive
14:15:05.915 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.113.126:44003(127.20.113.126:44003)] cleaning up while in state NEGOTIATION_FAILED due to: connection closed
14:15:05.917 [DEBUG - kudu-nio-0] (Connection.java:748) [peer master-127.20.113.126:44003(127.20.113.126:44003)] connecting to peer
14:15:05.920 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.20.113.126:44003(127.20.113.126:44003)] Successfully connected to peer
14:15:05.922 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.113.126:44003(127.20.113.126:44003)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials
14:15:05.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:15:05.925019 22166 negotiation.cc:337] Failed RPC negotiation. Trace:
14:15:05.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:15:05.918712 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:60886 (local address 127.20.113.126:44003)
14:15:05.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:15:05.919399 (+ 687us) server_negotiation.cc:207] Beginning negotiation
14:15:05.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:15:05.919423 (+ 24us) server_negotiation.cc:400] Waiting for connection header
14:15:05.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:15:05.921660 (+ 2237us) server_negotiation.cc:408] Connection header received
14:15:05.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:15:05.921815 (+ 155us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
14:15:05.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:15:05.921849 (+ 34us) server_negotiation.cc:462] Received NEGOTIATE request from client
14:15:05.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:15:05.922082 (+ 233us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
14:15:05.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:15:05.922265 (+ 183us) server_negotiation.cc:227] Negotiated authn=SASL
14:15:05.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:15:05.923796 (+ 1531us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:60886: BlockingRecv error: recv got EOF from 127.0.0.1:60886 (error 108)
14:15:05.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":146}
14:15:05.926 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.20.113.126:44003(127.20.113.126:44003)] handling channelInactive
14:15:05.928 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.113.126:44003: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials
14:15:05.928 [DEBUG - kudu-nio-0] (Connection.java:748) [peer master-127.20.113.125:43287(127.20.113.125:43287)] connecting to peer
14:15:05.929 [DEBUG - kudu-nio-0] (Connection.java:748) [peer master-127.20.113.124:34873(127.20.113.124:34873)] connecting to peer
14:15:05.932 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.20.113.124:34873(127.20.113.124:34873)] Successfully connected to peer
14:15:05.934 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.113.124:34873(127.20.113.124:34873)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials
14:15:05.935 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.20.113.124:34873(127.20.113.124:34873)] handling channelInactive
14:15:05.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:15:05.937461 22168 negotiation.cc:337] Failed RPC negotiation. Trace:
14:15:05.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:15:05.931237 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:46622 (local address 127.20.113.124:34873)
14:15:05.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:15:05.931979 (+ 742us) server_negotiation.cc:207] Beginning negotiation
14:15:05.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:15:05.932007 (+ 28us) server_negotiation.cc:400] Waiting for connection header
14:15:05.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:15:05.933284 (+ 1277us) server_negotiation.cc:408] Connection header received
14:15:05.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:15:05.933434 (+ 150us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
14:15:05.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:15:05.933461 (+ 27us) server_negotiation.cc:462] Received NEGOTIATE request from client
14:15:05.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:15:05.933705 (+ 244us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
14:15:05.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:15:05.933874 (+ 169us) server_negotiation.cc:227] Negotiated authn=SASL
14:15:05.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:15:05.935789 (+ 1915us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:46622: BlockingRecv error: recv got EOF from 127.0.0.1:46622 (error 108)
14:15:05.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":177}
14:15:05.938 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.113.124:34873: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials
14:15:05.945 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.20.113.125:43287(127.20.113.125:43287)] Successfully connected to peer
14:15:05.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:05.945703 22164 log.cc:826] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d: Log is configured to *not* fsync() on all Append() calls
14:15:05.956 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.113.125:43287(127.20.113.125:43287)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials
14:15:05.957 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.113.125:43287: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials
14:15:05.958 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.20.113.125:43287(127.20.113.125:43287)] disconnecting while in state TERMINATED
14:15:05.958 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.20.113.124:34873(127.20.113.124:34873)] disconnecting while in state TERMINATED
14:15:05.958 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.20.113.126:44003(127.20.113.126:44003)] disconnecting while in state TERMINATED
14:15:05.960 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:15:05.961 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.20.113.125:43287(127.20.113.125:43287)] handling channelInactive
14:15:05.962 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:15:05.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:15:05.966449 22169 negotiation.cc:337] Failed RPC negotiation. Trace:
14:15:05.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:15:05.939516 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:53360 (local address 127.20.113.125:43287)
14:15:05.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:15:05.955115 (+ 15599us) server_negotiation.cc:207] Beginning negotiation
14:15:05.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:15:05.955142 (+ 27us) server_negotiation.cc:400] Waiting for connection header
14:15:05.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:15:05.955189 (+ 47us) server_negotiation.cc:408] Connection header received
14:15:05.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:15:05.955350 (+ 161us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
14:15:05.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:15:05.955548 (+ 198us) server_negotiation.cc:462] Received NEGOTIATE request from client
14:15:05.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:15:05.955913 (+ 365us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
14:15:05.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:15:05.964596 (+ 8683us) server_negotiation.cc:227] Negotiated authn=SASL
14:15:05.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 14:15:05.965231 (+ 635us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:53360: BlockingRecv error: recv got EOF from 127.0.0.1:53360 (error 108)
14:15:05.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":15047,"thread_start_us":15828,"threads_started":1}
14:15:06.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:05.979703 20933 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu with pid 21890
14:15:06.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:05.980898 22164 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:15:06.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:05.981787 22164 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d: Bootstrap complete.
14:15:06.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:06.010303 22164 raft_consensus.cc:359] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } }
14:15:06.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:06.013687 22164 raft_consensus.cc:740] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 661b46d073c04359b178727bdbe67c4d, State: Initialized, Role: FOLLOWER
14:15:06.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:06.014618 22164 consensus_queue.cc:260] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } }
14:15:06.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:06.018445 22174 sys_catalog.cc:455] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } } }
14:15:06.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:06.020181 22164 sys_catalog.cc:565] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [sys.catalog]: configured and running, proceeding with master startup.
14:15:06.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:06.020877 22138 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [term 3 FOLLOWER]: Advancing to term 4
14:15:06.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:06.019812 22174 sys_catalog.cc:458] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [sys.catalog]: This master's current role is: FOLLOWER
14:15:06.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:06.040544 20933 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu with pid 21963
14:15:06.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 14:15:06.054373 22110 connection.cc:441] server connection from 127.0.0.1:46610 torn down before Call kudu.consensus.ConsensusService.UpdateConsensus from 127.0.0.1:46610 (request call id 0) could send its response
14:15:06.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:06.058396 22182 sys_catalog.cc:455] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "f7fa589d2c08468aa6cd09db9c033551" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } } }
14:15:06.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:06.059322 22182 sys_catalog.cc:458] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [sys.catalog]: This master's current role is: FOLLOWER
14:15:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:06.060021 22174 sys_catalog.cc:455] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [sys.catalog]: SysCatalogTable state changed. Reason: New leader f7fa589d2c08468aa6cd09db9c033551. Latest consensus state: current_term: 4 leader_uuid: "f7fa589d2c08468aa6cd09db9c033551" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f7fa589d2c08468aa6cd09db9c033551" member_type: VOTER last_known_addr { host: "127.20.113.126" port: 44003 } } peers { permanent_uuid: "70c2c7e4f63b4d3f94198427af0a5a3e" member_type: VOTER last_known_addr { host: "127.20.113.125" port: 43287 } } peers { permanent_uuid: "661b46d073c04359b178727bdbe67c4d" member_type: VOTER last_known_addr { host: "127.20.113.124" port: 34873 } } }
14:15:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:06.060732 22174 sys_catalog.cc:458] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d [sys.catalog]: This master's current role is: FOLLOWER
14:15:06.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:06.077476 22188 catalog_manager.cc:1269] Loaded cluster ID: e1b948fcf12a4a879df7c714938a923b
14:15:06.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:06.077729 22188 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 661b46d073c04359b178727bdbe67c4d: loading cluster ID for follower catalog manager: success
14:15:06.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:06.082226 20933 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu with pid 22058
14:15:07.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-25T14:15:07Z chronyd exiting
14:15:08.279 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:15:08.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:08.280189 20207 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu with pid 20488
14:15:08.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:08.309764 20207 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu with pid 20648
14:15:08.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:08.336148 20207 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu with pid 20792
14:15:08.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:08.360616 20207 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu with pid 20219
14:15:08.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:08.391865 20207 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu with pid 20289
14:15:08.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 14:15:08.422984 20207 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskB9qii1/build/tsan/bin/kudu with pid 20364
14:15:09.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-25T14:15:09Z chronyd exiting
14:15:10.544 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectExpires: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot re-acquire authentication token after 5 attempts (Couldn't find a valid master in (127.20.113.126:44003,127.20.113.125:43287,127.20.113.124:34873). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials])
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386) ~[test/:?]
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40) ~[kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectExpires(TestSecurity.java:412) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:158) ~[main/:?]
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:141) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: cannot re-acquire authentication token after 5 attempts (Couldn't find a valid master in (127.20.113.126:44003,127.20.113.125:43287,127.20.113.124:34873). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials])
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386)
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40)
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectExpires(TestSecurity.java:412)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:158)
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:141)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
14:15:11.347 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectExpires: giving up after 1 attempts
14:15:11.347 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectExpires: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot re-acquire authentication token after 5 attempts (Couldn't find a valid master in (127.20.113.126:44003,127.20.113.125:43287,127.20.113.124:34873). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials])
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386) ~[test/:?]
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40) ~[kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectExpires(TestSecurity.java:412) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:158) ~[main/:?]
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:141) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: cannot re-acquire authentication token after 5 attempts (Couldn't find a valid master in (127.20.113.126:44003,127.20.113.125:43287,127.20.113.124:34873). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials])
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386)
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40)
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectExpires(TestSecurity.java:412)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:158)
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:141)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more