08:07:47.699 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
08:07:47.699 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task_nU1WP/test-tmp
08:07:47.699 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
08:07:47.699 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task_nU1WP/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
08:07:47.702 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
08:07:48.018 [DEBUG - main] (MiniKuduCluster.java:195) Response:
08:07:48.018 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
08:07:48.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:48.021838 9065 mini_oidc.cc:150] Starting JWKS server
08:07:48.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:48.024919 9065 webserver.cc:492] Webserver started at http://127.0.0.1:46239/ using document root <none> and password file <none>
08:07:48.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:48.025326 9065 mini_oidc.cc:186] Starting OIDC Discovery server
08:07:48.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:48.026973 9065 webserver.cc:492] Webserver started at http://127.0.0.1:33283/ using document root <none> and password file <none>
08:07:48.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-03T08:07:48Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
08:07:48.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-03T08:07:48Z Disabled control of system clock
08:07:48.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:48.085865 9065 external_mini_cluster.cc:1366] Running /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu
08:07:48.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu
08:07:48.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-0/wal
08:07:48.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-0/data
08:07:48.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:07:48.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:07:48.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:07:48.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:48.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:48.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:48.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-0/logs
08:07:48.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-0/data/info.pb
08:07:48.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:48.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:07:48.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:07:48.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:07:48.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:07:48.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:48.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:48.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:48.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:07:48.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:07:48.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:48.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:48.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.218.126:39369
08:07:48.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.218.126
08:07:48.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:48.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.218.84:41765
08:07:48.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:48.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:48.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:48.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:48.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.218.126:39369,127.8.218.125:36841,127.8.218.124:43757
08:07:48.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:48.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46239/jwks.json with env {}
08:07:48.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:48.380400 9077 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:07:48.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:48.380944 9077 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:07:48.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:48.381400 9077 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:07:48.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:48.414214 9077 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:07:48.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:48.414621 9077 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:07:48.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:48.414971 9077 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:07:48.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:48.415298 9077 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:07:48.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:48.415643 9077 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:07:48.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:48.415915 9077 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:46239/jwks.json
08:07:48.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:48.454268 9077 master_runner.cc:386] Master server non-default flags:
08:07:48.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:48.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.218.84:41765
08:07:48.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:48.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:48.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-0/data
08:07:48.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-0/wal
08:07:48.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:48.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.218.126:39369,127.8.218.125:36841,127.8.218.124:43757
08:07:48.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:48.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:48.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:48.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.218.126:39369
08:07:48.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:48.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:07:48.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:48.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46239/jwks.json
08:07:48.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:48.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:48.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-0/data/info.pb
08:07:48.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.218.126
08:07:48.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:48.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:07:48.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:48.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:07:48.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:07:48.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:48.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-0/logs
08:07:48.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:48.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:07:48.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:07:48.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:07:48.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:07:48.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
08:07:48.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:07:48.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 07:43:21 UTC on e7f111948823
08:07:48.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11723
08:07:48.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:07:48.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:48.455863 9077 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:07:48.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:48.457451 9077 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:07:48.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:48.468269 9083 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:49.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:48.468444 9084 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:49.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:49.592350 9077 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.124s user 0.366s sys 0.755s
08:07:49.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:49.592911 9077 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.125s user 0.367s sys 0.755s
08:07:49.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:49.592950 9085 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1122 milliseconds
08:07:49.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:49.593705 9086 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:49.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:49.593797 9077 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:07:49.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:49.595319 9077 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:07:49.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:49.598341 9077 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:07:49.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:49.599892 9077 hybrid_clock.cc:648] HybridClock initialized: now 1777795669599841 us; error 37 us; skew 500 ppm
08:07:49.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:49.601019 9077 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:07:49.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:49.609970 9077 webserver.cc:492] Webserver started at http://127.8.218.126:40891/ using document root <none> and password file <none>
08:07:49.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:49.611395 9077 fs_manager.cc:362] Metadata directory not provided
08:07:49.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:49.611815 9077 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:07:49.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:49.612552 9077 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:07:49.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:49.618954 9077 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-0/data/instance:
08:07:49.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4afce85cbe814b07b34ec6b3c451c77d"
08:07:49.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 08:07:49 on dist-test-slave-tr3r"
08:07:49.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:49.620517 9077 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-0/wal/instance:
08:07:49.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4afce85cbe814b07b34ec6b3c451c77d"
08:07:49.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 08:07:49 on dist-test-slave-tr3r"
08:07:49.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:49.630234 9077 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.006s sys 0.006s
08:07:49.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:49.637800 9093 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:07:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:49.639144 9077 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
08:07:49.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:49.639564 9077 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-0/data,/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-0/wal
08:07:49.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4afce85cbe814b07b34ec6b3c451c77d"
08:07:49.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 08:07:49 on dist-test-slave-tr3r"
08:07:49.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:49.639994 9077 fs_report.cc:389] FS layout report
08:07:49.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:07:49.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-0/wal
08:07:49.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-0/wal
08:07:49.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-0/data/data
08:07:49.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:07:49.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:07:49.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:07:49.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:07:49.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:07:49.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:07:49.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:07:49.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:07:49.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:07:49.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:07:49.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:07:49.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:49.716805 9077 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:07:49.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:49.718345 9077 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:07:49.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:49.718794 9077 kserver.cc:163] Server-wide thread pool size limit: 3276
08:07:49.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:49.787958 9077 rpc_server.cc:307] RPC server started. Bound to: 127.8.218.126:39369
08:07:49.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:49.788028 9144 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.218.126:39369 every 8 connection(s)
08:07:49.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:49.790663 9077 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-0/data/info.pb
08:07:49.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:49.796566 9145 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:07:49.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:49.798080 9065 external_mini_cluster.cc:1428] Started /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu as pid 9077
08:07:49.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:49.798525 9065 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-0/wal/instance
08:07:49.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:49.801885 9065 external_mini_cluster.cc:1366] Running /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu
08:07:49.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu
08:07:49.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-1/wal
08:07:49.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-1/data
08:07:49.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:07:49.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:07:49.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:07:49.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:49.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:49.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:49.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-1/logs
08:07:49.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-1/data/info.pb
08:07:49.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:49.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:07:49.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:07:49.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:07:49.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:07:49.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:49.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:49.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:49.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:07:49.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:07:49.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:49.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:49.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.218.125:36841
08:07:49.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.218.125
08:07:49.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:49.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.218.84:41765
08:07:49.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:49.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:49.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:49.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:49.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.218.126:39369,127.8.218.125:36841,127.8.218.124:43757
08:07:49.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:49.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46239/jwks.json with env {}
08:07:49.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:49.806825 9145 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.218.126" port: 39369 } has no permanent_uuid. Determining permanent_uuid...
08:07:49.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:49.833961 9145 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.218.125" port: 36841 } has no permanent_uuid. Determining permanent_uuid...
08:07:49.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:49.836283 9096 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.218.125:36841: connect: Connection refused (error 111)
08:07:49.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:49.840126 9145 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.218.125:36841: Network error: Client connection negotiation failed: client connection to 127.8.218.125:36841: connect: Connection refused (error 111)
08:07:49.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:49.889737 9145 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.218.125" port: 36841 } attempt: 1
08:07:49.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:49.893566 9145 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.218.125:36841: Network error: Client connection negotiation failed: client connection to 127.8.218.125:36841: connect: Connection refused (error 111)
08:07:49.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:49.961948 9145 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.218.125" port: 36841 } attempt: 2
08:07:49.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:49.965735 9145 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.218.125:36841: Network error: Client connection negotiation failed: client connection to 127.8.218.125:36841: connect: Connection refused (error 111)
08:07:50.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:50.057217 9145 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.218.125" port: 36841 } attempt: 3
08:07:50.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:50.060952 9145 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.218.125:36841: Network error: Client connection negotiation failed: client connection to 127.8.218.125:36841: connect: Connection refused (error 111)
08:07:50.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:50.100178 9147 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:07:50.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:50.100726 9147 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:07:50.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:50.101162 9147 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:07:50.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:50.134272 9147 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:07:50.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:50.134562 9147 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:07:50.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:50.134806 9147 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:07:50.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:50.135016 9147 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:07:50.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:50.135221 9147 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:07:50.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:50.135370 9147 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:46239/jwks.json
08:07:50.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:50.172523 9147 master_runner.cc:386] Master server non-default flags:
08:07:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.218.84:41765
08:07:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-1/data
08:07:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-1/wal
08:07:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.218.126:39369,127.8.218.125:36841,127.8.218.124:43757
08:07:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.218.125:36841
08:07:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:07:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46239/jwks.json
08:07:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-1/data/info.pb
08:07:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.218.125
08:07:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:07:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:07:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:07:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-1/logs
08:07:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:07:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:07:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:07:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:07:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
08:07:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:07:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 07:43:21 UTC on e7f111948823
08:07:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11723
08:07:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:07:50.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:50.173863 9147 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:07:50.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:50.175486 9147 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:07:50.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:50.187021 9156 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:50.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:50.204406 9145 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.218.125" port: 36841 } attempt: 4
08:07:50.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:50.210501 9145 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.218.125:36841: Network error: Client connection negotiation failed: client connection to 127.8.218.125:36841: connect: Connection refused (error 111)
08:07:50.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:50.509842 9145 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.218.125" port: 36841 } attempt: 5
08:07:50.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:50.516814 9145 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.218.125:36841: Network error: Client connection negotiation failed: client connection to 127.8.218.125:36841: connect: Connection refused (error 111)
08:07:51.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:51.064322 9145 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.218.125" port: 36841 } attempt: 6
08:07:51.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:51.093026 9145 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.218.125:36841: Network error: Client connection negotiation failed: client connection to 127.8.218.125:36841: connect: Connection refused (error 111)
08:07:51.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:50.187538 9157 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:51.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:51.373085 9159 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:51.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:51.375273 9147 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.189s user 0.451s sys 0.727s
08:07:51.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:51.375865 9147 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.189s user 0.452s sys 0.728s
08:07:51.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:51.376026 9158 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1183 milliseconds
08:07:51.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:51.376163 9147 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:07:51.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:51.377648 9147 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:07:51.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:51.380769 9147 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:07:51.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:51.382225 9147 hybrid_clock.cc:648] HybridClock initialized: now 1777795671382148 us; error 78 us; skew 500 ppm
08:07:51.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:51.383311 9147 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:07:51.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:51.391773 9147 webserver.cc:492] Webserver started at http://127.8.218.125:39007/ using document root <none> and password file <none>
08:07:51.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:51.393122 9147 fs_manager.cc:362] Metadata directory not provided
08:07:51.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:51.393420 9147 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:07:51.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:51.393996 9147 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:07:51.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:51.400302 9147 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-1/data/instance:
08:07:51.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "88020bf871104cab8c5dd634c3f6f55d"
08:07:51.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 08:07:51 on dist-test-slave-tr3r"
08:07:51.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:51.401813 9147 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-1/wal/instance:
08:07:51.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "88020bf871104cab8c5dd634c3f6f55d"
08:07:51.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 08:07:51 on dist-test-slave-tr3r"
08:07:51.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:51.411386 9147 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.006s sys 0.002s
08:07:51.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:51.418808 9168 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:07:51.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:51.420050 9147 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.003s
08:07:51.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:51.420449 9147 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-1/data,/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-1/wal
08:07:51.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "88020bf871104cab8c5dd634c3f6f55d"
08:07:51.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 08:07:51 on dist-test-slave-tr3r"
08:07:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:51.420861 9147 fs_report.cc:389] FS layout report
08:07:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:07:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-1/wal
08:07:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-1/wal
08:07:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-1/data/data
08:07:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:07:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:07:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:07:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:07:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:07:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:07:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:07:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:07:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:07:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:07:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:07:51.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:51.528259 9147 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:07:51.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:51.529768 9147 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:07:51.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:51.530195 9147 kserver.cc:163] Server-wide thread pool size limit: 3276
08:07:51.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:51.598131 9147 rpc_server.cc:307] RPC server started. Bound to: 127.8.218.125:36841
08:07:51.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:51.598214 9219 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.218.125:36841 every 8 connection(s)
08:07:51.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:51.600651 9147 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-1/data/info.pb
08:07:51.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:51.605530 9065 external_mini_cluster.cc:1428] Started /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu as pid 9147
08:07:51.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:51.605990 9065 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-1/wal/instance
08:07:51.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:51.606452 9220 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:07:51.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:51.608834 9065 external_mini_cluster.cc:1366] Running /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu
08:07:51.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu
08:07:51.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-2/wal
08:07:51.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-2/data
08:07:51.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:07:51.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:07:51.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:07:51.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:51.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:51.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:51.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-2/logs
08:07:51.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-2/data/info.pb
08:07:51.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:51.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:07:51.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:07:51.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:07:51.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:07:51.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:51.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:51.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:51.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:07:51.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:07:51.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:51.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:51.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.218.124:43757
08:07:51.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.218.124
08:07:51.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:51.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.218.84:41765
08:07:51.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:51.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:51.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:51.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:51.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.218.126:39369,127.8.218.125:36841,127.8.218.124:43757
08:07:51.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:51.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46239/jwks.json with env {}
08:07:51.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:51.618894 9220 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.218.126" port: 39369 } has no permanent_uuid. Determining permanent_uuid...
08:07:51.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:51.639088 9220 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.218.125" port: 36841 } has no permanent_uuid. Determining permanent_uuid...
08:07:51.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:51.653527 9220 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.218.124" port: 43757 } has no permanent_uuid. Determining permanent_uuid...
08:07:51.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:51.655859 9170 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.218.124:43757: connect: Connection refused (error 111)
08:07:51.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:51.659881 9220 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.218.124:43757: Network error: Client connection negotiation failed: client connection to 127.8.218.124:43757: connect: Connection refused (error 111)
08:07:51.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:51.709314 9220 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.218.124" port: 43757 } attempt: 1
08:07:51.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:51.713528 9220 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.218.124:43757: Network error: Client connection negotiation failed: client connection to 127.8.218.124:43757: connect: Connection refused (error 111)
08:07:51.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:51.781957 9220 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.218.124" port: 43757 } attempt: 2
08:07:51.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:51.786005 9220 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.218.124:43757: Network error: Client connection negotiation failed: client connection to 127.8.218.124:43757: connect: Connection refused (error 111)
08:07:51.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:51.877462 9220 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.218.124" port: 43757 } attempt: 3
08:07:51.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:51.881817 9220 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.218.124:43757: Network error: Client connection negotiation failed: client connection to 127.8.218.124:43757: connect: Connection refused (error 111)
08:07:51.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:51.917999 9222 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:07:51.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:51.918555 9222 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:07:51.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:51.919016 9222 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:07:51.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:51.954088 9222 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:07:51.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:51.954394 9222 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:07:51.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:51.954607 9222 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:07:51.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:51.954855 9222 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:07:51.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:51.955066 9222 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:07:51.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:51.955230 9222 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:46239/jwks.json
08:07:51.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:51.992250 9222 master_runner.cc:386] Master server non-default flags:
08:07:51.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:51.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.218.84:41765
08:07:51.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:51.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:51.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-2/data
08:07:51.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-2/wal
08:07:51.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:51.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.218.126:39369,127.8.218.125:36841,127.8.218.124:43757
08:07:51.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:51.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:51.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:51.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.218.124:43757
08:07:51.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:51.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:07:51.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:51.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46239/jwks.json
08:07:51.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:51.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:51.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-2/data/info.pb
08:07:51.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.218.124
08:07:51.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:51.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:07:51.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:51.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:07:51.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:07:51.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:51.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-2/logs
08:07:51.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:51.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:07:51.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:07:51.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:07:51.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:07:51.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
08:07:51.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:07:51.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 07:43:21 UTC on e7f111948823
08:07:51.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11723
08:07:51.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:07:51.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:51.993624 9222 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:07:51.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:51.995250 9222 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:07:52.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:52.006265 9232 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:52.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:52.025228 9220 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.218.124" port: 43757 } attempt: 4
08:07:52.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:52.031237 9220 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.218.124:43757: Network error: Client connection negotiation failed: client connection to 127.8.218.124:43757: connect: Connection refused (error 111)
08:07:52.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:52.153529 9145 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.218.125" port: 36841 } attempt: 7
08:07:52.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:52.194410 9145 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.218.124" port: 43757 } has no permanent_uuid. Determining permanent_uuid...
08:07:52.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:52.212411 9145 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.218.124:43757: Network error: Client connection negotiation failed: client connection to 127.8.218.124:43757: connect: Connection refused (error 111)
08:07:52.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:52.270880 9145 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.218.124" port: 43757 } attempt: 1
08:07:52.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:52.277029 9145 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.218.124:43757: Network error: Client connection negotiation failed: client connection to 127.8.218.124:43757: connect: Connection refused (error 111)
08:07:52.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:52.330703 9220 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.218.124" port: 43757 } attempt: 5
08:07:52.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:52.337812 9220 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.218.124:43757: Network error: Client connection negotiation failed: client connection to 127.8.218.124:43757: connect: Connection refused (error 111)
08:07:52.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:52.358500 9145 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.218.124" port: 43757 } attempt: 2
08:07:52.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:52.364485 9145 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.218.124:43757: Network error: Client connection negotiation failed: client connection to 127.8.218.124:43757: connect: Connection refused (error 111)
08:07:52.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:52.449945 9145 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.218.124" port: 43757 } attempt: 3
08:07:52.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:52.456039 9145 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.218.124:43757: Network error: Client connection negotiation failed: client connection to 127.8.218.124:43757: connect: Connection refused (error 111)
08:07:52.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:52.600762 9145 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.218.124" port: 43757 } attempt: 4
08:07:52.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:52.617566 9145 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.218.124:43757: Network error: Client connection negotiation failed: client connection to 127.8.218.124:43757: connect: Connection refused (error 111)
08:07:52.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:52.885368 9220 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.218.124" port: 43757 } attempt: 6
08:07:52.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:52.901054 9145 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.218.124" port: 43757 } attempt: 5
08:07:52.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:52.907018 9145 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.218.124:43757: Network error: Client connection negotiation failed: client connection to 127.8.218.124:43757: connect: Connection refused (error 111)
08:07:52.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:52.917887 9220 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.218.124:43757: Network error: Client connection negotiation failed: client connection to 127.8.218.124:43757: connect: Connection refused (error 111)
08:07:53.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:52.006927 9233 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:53.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:53.336551 9235 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:53.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:53.339248 9234 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1332 milliseconds
08:07:53.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:53.339700 9222 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.333s user 0.536s sys 0.792s
08:07:53.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:53.340023 9222 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.334s user 0.536s sys 0.792s
08:07:53.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:53.340297 9222 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:07:53.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:53.341364 9222 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:07:53.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:53.344003 9222 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:07:53.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:53.345381 9222 hybrid_clock.cc:648] HybridClock initialized: now 1777795673345335 us; error 61 us; skew 500 ppm
08:07:53.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:53.346246 9222 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:07:53.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:53.353624 9222 webserver.cc:492] Webserver started at http://127.8.218.124:35007/ using document root <none> and password file <none>
08:07:53.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:53.354633 9222 fs_manager.cc:362] Metadata directory not provided
08:07:53.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:53.354862 9222 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:07:53.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:53.355345 9222 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:07:53.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:53.359701 9222 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-2/data/instance:
08:07:53.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "24ef2e832f624a8cb3741d753b948b9f"
08:07:53.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 08:07:53 on dist-test-slave-tr3r"
08:07:53.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:53.360816 9222 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-2/wal/instance:
08:07:53.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "24ef2e832f624a8cb3741d753b948b9f"
08:07:53.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 08:07:53 on dist-test-slave-tr3r"
08:07:53.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:53.368526 9222 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.001s sys 0.009s
08:07:53.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:53.375195 9247 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:07:53.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:53.376540 9222 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
08:07:53.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:53.376885 9222 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-2/data,/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-2/wal
08:07:53.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "24ef2e832f624a8cb3741d753b948b9f"
08:07:53.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 08:07:53 on dist-test-slave-tr3r"
08:07:53.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:53.377238 9222 fs_report.cc:389] FS layout report
08:07:53.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:07:53.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-2/wal
08:07:53.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-2/wal
08:07:53.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-2/data/data
08:07:53.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:07:53.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:07:53.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:07:53.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:07:53.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:07:53.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:07:53.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:07:53.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:07:53.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:07:53.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:07:53.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:07:53.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:53.451386 9222 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:07:53.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:53.452955 9222 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:07:53.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:53.453415 9222 kserver.cc:163] Server-wide thread pool size limit: 3276
08:07:53.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:53.459520 9145 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.218.124" port: 43757 } attempt: 6
08:07:53.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:53.466285 9145 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.218.124:43757: Network error: Client connection negotiation failed: client connection to 127.8.218.124:43757: connect: Connection refused (error 111)
08:07:53.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:53.523263 9222 rpc_server.cc:307] RPC server started. Bound to: 127.8.218.124:43757
08:07:53.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:53.523371 9300 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.218.124:43757 every 8 connection(s)
08:07:53.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:53.526014 9222 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-2/data/info.pb
08:07:53.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:53.531697 9301 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:07:53.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:53.532302 9065 external_mini_cluster.cc:1428] Started /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu as pid 9222
08:07:53.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:53.532691 9065 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/master-2/wal/instance
08:07:53.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:53.545037 9301 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.218.126" port: 39369 } has no permanent_uuid. Determining permanent_uuid...
08:07:53.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:53.568898 9301 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.218.125" port: 36841 } has no permanent_uuid. Determining permanent_uuid...
08:07:53.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:53.577818 9301 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.218.124" port: 43757 } has no permanent_uuid. Determining permanent_uuid...
08:07:53.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:53.593628 9301 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 24ef2e832f624a8cb3741d753b948b9f: Bootstrap starting.
08:07:53.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:53.600678 9301 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 24ef2e832f624a8cb3741d753b948b9f: Neither blocks nor log segments found. Creating new log.
08:07:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:53.602391 9301 log.cc:826] T 00000000000000000000000000000000 P 24ef2e832f624a8cb3741d753b948b9f: Log is configured to *not* fsync() on all Append() calls
08:07:53.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:53.607038 9301 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 24ef2e832f624a8cb3741d753b948b9f: No bootstrap required, opened a new log
08:07:53.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:53.625140 9301 raft_consensus.cc:359] T 00000000000000000000000000000000 P 24ef2e832f624a8cb3741d753b948b9f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4afce85cbe814b07b34ec6b3c451c77d" member_type: VOTER last_known_addr { host: "127.8.218.126" port: 39369 } } peers { permanent_uuid: "88020bf871104cab8c5dd634c3f6f55d" member_type: VOTER last_known_addr { host: "127.8.218.125" port: 36841 } } peers { permanent_uuid: "24ef2e832f624a8cb3741d753b948b9f" member_type: VOTER last_known_addr { host: "127.8.218.124" port: 43757 } }
08:07:53.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:53.625849 9301 raft_consensus.cc:385] T 00000000000000000000000000000000 P 24ef2e832f624a8cb3741d753b948b9f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:07:53.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:53.626114 9301 raft_consensus.cc:740] T 00000000000000000000000000000000 P 24ef2e832f624a8cb3741d753b948b9f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 24ef2e832f624a8cb3741d753b948b9f, State: Initialized, Role: FOLLOWER
08:07:53.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:53.626909 9301 consensus_queue.cc:260] T 00000000000000000000000000000000 P 24ef2e832f624a8cb3741d753b948b9f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4afce85cbe814b07b34ec6b3c451c77d" member_type: VOTER last_known_addr { host: "127.8.218.126" port: 39369 } } peers { permanent_uuid: "88020bf871104cab8c5dd634c3f6f55d" member_type: VOTER last_known_addr { host: "127.8.218.125" port: 36841 } } peers { permanent_uuid: "24ef2e832f624a8cb3741d753b948b9f" member_type: VOTER last_known_addr { host: "127.8.218.124" port: 43757 } }
08:07:53.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:53.629305 9310 sys_catalog.cc:455] T 00000000000000000000000000000000 P 24ef2e832f624a8cb3741d753b948b9f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4afce85cbe814b07b34ec6b3c451c77d" member_type: VOTER last_known_addr { host: "127.8.218.126" port: 39369 } } peers { permanent_uuid: "88020bf871104cab8c5dd634c3f6f55d" member_type: VOTER last_known_addr { host: "127.8.218.125" port: 36841 } } peers { permanent_uuid: "24ef2e832f624a8cb3741d753b948b9f" member_type: VOTER last_known_addr { host: "127.8.218.124" port: 43757 } } }
08:07:53.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:53.629881 9310 sys_catalog.cc:458] T 00000000000000000000000000000000 P 24ef2e832f624a8cb3741d753b948b9f [sys.catalog]: This master's current role is: FOLLOWER
08:07:53.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:53.630700 9301 sys_catalog.cc:565] T 00000000000000000000000000000000 P 24ef2e832f624a8cb3741d753b948b9f [sys.catalog]: configured and running, proceeding with master startup.
08:07:53.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:53.654822 9321 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 24ef2e832f624a8cb3741d753b948b9f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:07:53.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:53.655164 9321 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:07:53.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:53.648752 9222 master_runner.cc:186] Error getting master registration for 127.8.218.126:39369: OK, instance_id { permanent_uuid: "4afce85cbe814b07b34ec6b3c451c77d" instance_seqno: 1777795669769025 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:07:53.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:53.670128 9222 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:07:53.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:53.708494 9253 tablet.cc:2404] T 00000000000000000000000000000000 P 24ef2e832f624a8cb3741d753b948b9f: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:07:53.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:53.879549 9310 raft_consensus.cc:493] T 00000000000000000000000000000000 P 24ef2e832f624a8cb3741d753b948b9f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:07:53.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:53.880020 9310 raft_consensus.cc:515] T 00000000000000000000000000000000 P 24ef2e832f624a8cb3741d753b948b9f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4afce85cbe814b07b34ec6b3c451c77d" member_type: VOTER last_known_addr { host: "127.8.218.126" port: 39369 } } peers { permanent_uuid: "88020bf871104cab8c5dd634c3f6f55d" member_type: VOTER last_known_addr { host: "127.8.218.125" port: 36841 } } peers { permanent_uuid: "24ef2e832f624a8cb3741d753b948b9f" member_type: VOTER last_known_addr { host: "127.8.218.124" port: 43757 } }
08:07:53.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:53.882081 9310 leader_election.cc:290] T 00000000000000000000000000000000 P 24ef2e832f624a8cb3741d753b948b9f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4afce85cbe814b07b34ec6b3c451c77d (127.8.218.126:39369), 88020bf871104cab8c5dd634c3f6f55d (127.8.218.125:36841)
08:07:53.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:53.978312 9220 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.218.124" port: 43757 } attempt: 7
08:07:54.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.001724 9220 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 88020bf871104cab8c5dd634c3f6f55d: Bootstrap starting.
08:07:54.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.009400 9220 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 88020bf871104cab8c5dd634c3f6f55d: Neither blocks nor log segments found. Creating new log.
08:07:54.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.011807 9220 log.cc:826] T 00000000000000000000000000000000 P 88020bf871104cab8c5dd634c3f6f55d: Log is configured to *not* fsync() on all Append() calls
08:07:54.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.016844 9220 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 88020bf871104cab8c5dd634c3f6f55d: No bootstrap required, opened a new log
08:07:54.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.038811 9220 raft_consensus.cc:359] T 00000000000000000000000000000000 P 88020bf871104cab8c5dd634c3f6f55d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4afce85cbe814b07b34ec6b3c451c77d" member_type: VOTER last_known_addr { host: "127.8.218.126" port: 39369 } } peers { permanent_uuid: "88020bf871104cab8c5dd634c3f6f55d" member_type: VOTER last_known_addr { host: "127.8.218.125" port: 36841 } } peers { permanent_uuid: "24ef2e832f624a8cb3741d753b948b9f" member_type: VOTER last_known_addr { host: "127.8.218.124" port: 43757 } }
08:07:54.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.039534 9220 raft_consensus.cc:385] T 00000000000000000000000000000000 P 88020bf871104cab8c5dd634c3f6f55d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:07:54.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.039940 9220 raft_consensus.cc:740] T 00000000000000000000000000000000 P 88020bf871104cab8c5dd634c3f6f55d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 88020bf871104cab8c5dd634c3f6f55d, State: Initialized, Role: FOLLOWER
08:07:54.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.041149 9220 consensus_queue.cc:260] T 00000000000000000000000000000000 P 88020bf871104cab8c5dd634c3f6f55d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4afce85cbe814b07b34ec6b3c451c77d" member_type: VOTER last_known_addr { host: "127.8.218.126" port: 39369 } } peers { permanent_uuid: "88020bf871104cab8c5dd634c3f6f55d" member_type: VOTER last_known_addr { host: "127.8.218.125" port: 36841 } } peers { permanent_uuid: "24ef2e832f624a8cb3741d753b948b9f" member_type: VOTER last_known_addr { host: "127.8.218.124" port: 43757 } }
08:07:54.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.045030 9325 sys_catalog.cc:455] T 00000000000000000000000000000000 P 88020bf871104cab8c5dd634c3f6f55d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4afce85cbe814b07b34ec6b3c451c77d" member_type: VOTER last_known_addr { host: "127.8.218.126" port: 39369 } } peers { permanent_uuid: "88020bf871104cab8c5dd634c3f6f55d" member_type: VOTER last_known_addr { host: "127.8.218.125" port: 36841 } } peers { permanent_uuid: "24ef2e832f624a8cb3741d753b948b9f" member_type: VOTER last_known_addr { host: "127.8.218.124" port: 43757 } } }
08:07:54.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.046617 9325 sys_catalog.cc:458] T 00000000000000000000000000000000 P 88020bf871104cab8c5dd634c3f6f55d [sys.catalog]: This master's current role is: FOLLOWER
08:07:54.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.048096 9220 sys_catalog.cc:565] T 00000000000000000000000000000000 P 88020bf871104cab8c5dd634c3f6f55d [sys.catalog]: configured and running, proceeding with master startup.
08:07:54.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.048316 9195 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "24ef2e832f624a8cb3741d753b948b9f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "88020bf871104cab8c5dd634c3f6f55d" is_pre_election: true
08:07:54.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.049129 9195 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 88020bf871104cab8c5dd634c3f6f55d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 24ef2e832f624a8cb3741d753b948b9f in term 0.
08:07:54.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.050714 9250 leader_election.cc:304] T 00000000000000000000000000000000 P 24ef2e832f624a8cb3741d753b948b9f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 24ef2e832f624a8cb3741d753b948b9f, 88020bf871104cab8c5dd634c3f6f55d; no voters:
08:07:54.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.052182 9310 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 24ef2e832f624a8cb3741d753b948b9f [term 0 FOLLOWER]: Leader pre-election won for term 1
08:07:54.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.052531 9310 raft_consensus.cc:493] T 00000000000000000000000000000000 P 24ef2e832f624a8cb3741d753b948b9f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:07:54.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.052819 9310 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 24ef2e832f624a8cb3741d753b948b9f [term 0 FOLLOWER]: Advancing to term 1
08:07:54.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.057073 9310 raft_consensus.cc:515] T 00000000000000000000000000000000 P 24ef2e832f624a8cb3741d753b948b9f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4afce85cbe814b07b34ec6b3c451c77d" member_type: VOTER last_known_addr { host: "127.8.218.126" port: 39369 } } peers { permanent_uuid: "88020bf871104cab8c5dd634c3f6f55d" member_type: VOTER last_known_addr { host: "127.8.218.125" port: 36841 } } peers { permanent_uuid: "24ef2e832f624a8cb3741d753b948b9f" member_type: VOTER last_known_addr { host: "127.8.218.124" port: 43757 } }
08:07:54.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.059970 9310 leader_election.cc:290] T 00000000000000000000000000000000 P 24ef2e832f624a8cb3741d753b948b9f [CANDIDATE]: Term 1 election: Requested vote from peers 4afce85cbe814b07b34ec6b3c451c77d (127.8.218.126:39369), 88020bf871104cab8c5dd634c3f6f55d (127.8.218.125:36841)
08:07:54.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.060992 9195 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "24ef2e832f624a8cb3741d753b948b9f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "88020bf871104cab8c5dd634c3f6f55d"
08:07:54.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.061528 9195 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 88020bf871104cab8c5dd634c3f6f55d [term 0 FOLLOWER]: Advancing to term 1
08:07:54.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.073280 9195 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 88020bf871104cab8c5dd634c3f6f55d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 24ef2e832f624a8cb3741d753b948b9f in term 1.
08:07:54.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.075817 9250 leader_election.cc:304] T 00000000000000000000000000000000 P 24ef2e832f624a8cb3741d753b948b9f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 24ef2e832f624a8cb3741d753b948b9f, 88020bf871104cab8c5dd634c3f6f55d; no voters:
08:07:54.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.076934 9310 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 24ef2e832f624a8cb3741d753b948b9f [term 1 FOLLOWER]: Leader election won for term 1
08:07:54.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.078776 9310 raft_consensus.cc:697] T 00000000000000000000000000000000 P 24ef2e832f624a8cb3741d753b948b9f [term 1 LEADER]: Becoming Leader. State: Replica: 24ef2e832f624a8cb3741d753b948b9f, State: Running, Role: LEADER
08:07:54.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.079864 9310 consensus_queue.cc:237] T 00000000000000000000000000000000 P 24ef2e832f624a8cb3741d753b948b9f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4afce85cbe814b07b34ec6b3c451c77d" member_type: VOTER last_known_addr { host: "127.8.218.126" port: 39369 } } peers { permanent_uuid: "88020bf871104cab8c5dd634c3f6f55d" member_type: VOTER last_known_addr { host: "127.8.218.125" port: 36841 } } peers { permanent_uuid: "24ef2e832f624a8cb3741d753b948b9f" member_type: VOTER last_known_addr { host: "127.8.218.124" port: 43757 } }
08:07:54.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.099167 9331 sys_catalog.cc:455] T 00000000000000000000000000000000 P 24ef2e832f624a8cb3741d753b948b9f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 24ef2e832f624a8cb3741d753b948b9f. Latest consensus state: current_term: 1 leader_uuid: "24ef2e832f624a8cb3741d753b948b9f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4afce85cbe814b07b34ec6b3c451c77d" member_type: VOTER last_known_addr { host: "127.8.218.126" port: 39369 } } peers { permanent_uuid: "88020bf871104cab8c5dd634c3f6f55d" member_type: VOTER last_known_addr { host: "127.8.218.125" port: 36841 } } peers { permanent_uuid: "24ef2e832f624a8cb3741d753b948b9f" member_type: VOTER last_known_addr { host: "127.8.218.124" port: 43757 } } }
08:07:54.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.100157 9331 sys_catalog.cc:458] T 00000000000000000000000000000000 P 24ef2e832f624a8cb3741d753b948b9f [sys.catalog]: This master's current role is: LEADER
08:07:54.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.111963 9339 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
08:07:54.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:54.120596 9338 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 88020bf871104cab8c5dd634c3f6f55d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:07:54.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:54.120962 9338 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:07:54.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.102914 9147 master_runner.cc:186] Error getting master registration for 127.8.218.126:39369: OK, instance_id { permanent_uuid: "4afce85cbe814b07b34ec6b3c451c77d" instance_seqno: 1777795669769025 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:07:54.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.133180 9147 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:07:54.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.136468 9339 catalog_manager.cc:1494] Initializing Kudu cluster ID...
08:07:54.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.155076 9195 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 88020bf871104cab8c5dd634c3f6f55d [term 1 FOLLOWER]: Refusing update from remote peer 24ef2e832f624a8cb3741d753b948b9f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:07:54.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.156682 9310 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 24ef2e832f624a8cb3741d753b948b9f [LEADER]: Connected to new peer: Peer: permanent_uuid: "88020bf871104cab8c5dd634c3f6f55d" member_type: VOTER last_known_addr { host: "127.8.218.125" port: 36841 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:07:54.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.186609 9325 sys_catalog.cc:455] T 00000000000000000000000000000000 P 88020bf871104cab8c5dd634c3f6f55d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 24ef2e832f624a8cb3741d753b948b9f. Latest consensus state: current_term: 1 leader_uuid: "24ef2e832f624a8cb3741d753b948b9f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4afce85cbe814b07b34ec6b3c451c77d" member_type: VOTER last_known_addr { host: "127.8.218.126" port: 39369 } } peers { permanent_uuid: "88020bf871104cab8c5dd634c3f6f55d" member_type: VOTER last_known_addr { host: "127.8.218.125" port: 36841 } } peers { permanent_uuid: "24ef2e832f624a8cb3741d753b948b9f" member_type: VOTER last_known_addr { host: "127.8.218.124" port: 43757 } } }
08:07:54.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.191430 9325 sys_catalog.cc:458] T 00000000000000000000000000000000 P 88020bf871104cab8c5dd634c3f6f55d [sys.catalog]: This master's current role is: FOLLOWER
08:07:54.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.196856 9331 sys_catalog.cc:455] T 00000000000000000000000000000000 P 24ef2e832f624a8cb3741d753b948b9f [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "24ef2e832f624a8cb3741d753b948b9f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4afce85cbe814b07b34ec6b3c451c77d" member_type: VOTER last_known_addr { host: "127.8.218.126" port: 39369 } } peers { permanent_uuid: "88020bf871104cab8c5dd634c3f6f55d" member_type: VOTER last_known_addr { host: "127.8.218.125" port: 36841 } } peers { permanent_uuid: "24ef2e832f624a8cb3741d753b948b9f" member_type: VOTER last_known_addr { host: "127.8.218.124" port: 43757 } } }
08:07:54.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.197590 9331 sys_catalog.cc:458] T 00000000000000000000000000000000 P 24ef2e832f624a8cb3741d753b948b9f [sys.catalog]: This master's current role is: LEADER
08:07:54.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.201002 9342 mvcc.cc:204] Tried to move back new op lower bound from 7281851081320910848 to 7281851081051357184. Current Snapshot: MvccSnapshot[applied={T|T < 7281851081320910848}]
08:07:54.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.207738 9339 catalog_manager.cc:1357] Generated new cluster ID: 4bfe83002d3344f9b16c27f2d2fedbee
08:07:54.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.208057 9339 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
08:07:54.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.208521 9325 sys_catalog.cc:455] T 00000000000000000000000000000000 P 88020bf871104cab8c5dd634c3f6f55d [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "24ef2e832f624a8cb3741d753b948b9f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4afce85cbe814b07b34ec6b3c451c77d" member_type: VOTER last_known_addr { host: "127.8.218.126" port: 39369 } } peers { permanent_uuid: "88020bf871104cab8c5dd634c3f6f55d" member_type: VOTER last_known_addr { host: "127.8.218.125" port: 36841 } } peers { permanent_uuid: "24ef2e832f624a8cb3741d753b948b9f" member_type: VOTER last_known_addr { host: "127.8.218.124" port: 43757 } } }
08:07:54.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.209177 9325 sys_catalog.cc:458] T 00000000000000000000000000000000 P 88020bf871104cab8c5dd634c3f6f55d [sys.catalog]: This master's current role is: FOLLOWER
08:07:54.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.273284 9339 catalog_manager.cc:1380] Generated new certificate authority record
08:07:54.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.277307 9339 catalog_manager.cc:1514] Loading token signing keys...
08:07:54.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.301347 9339 catalog_manager.cc:6044] T 00000000000000000000000000000000 P 24ef2e832f624a8cb3741d753b948b9f: Generated new TSK 0
08:07:54.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.302914 9339 catalog_manager.cc:1524] Initializing in-progress tserver states...
08:07:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.313898 9065 external_mini_cluster.cc:1366] Running /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu
08:07:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu
08:07:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-0/wal
08:07:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-0/data
08:07:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:07:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:07:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:07:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-0/logs
08:07:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-0/data/info.pb
08:07:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:07:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:07:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:07:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:07:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:07:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:07:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.218.65:0
08:07:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.218.65
08:07:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.218.65
08:07:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.218.126:39369,127.8.218.125:36841,127.8.218.124:43757
08:07:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.218.84:41765
08:07:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:07:54.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.499722 9145 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.218.124" port: 43757 } attempt: 7
08:07:54.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.554489 9145 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4afce85cbe814b07b34ec6b3c451c77d: Bootstrap starting.
08:07:54.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.574277 9145 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 4afce85cbe814b07b34ec6b3c451c77d: Neither blocks nor log segments found. Creating new log.
08:07:54.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.581856 9145 log.cc:826] T 00000000000000000000000000000000 P 4afce85cbe814b07b34ec6b3c451c77d: Log is configured to *not* fsync() on all Append() calls
08:07:54.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.605170 9145 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4afce85cbe814b07b34ec6b3c451c77d: No bootstrap required, opened a new log
08:07:54.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.684430 9145 raft_consensus.cc:359] T 00000000000000000000000000000000 P 4afce85cbe814b07b34ec6b3c451c77d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4afce85cbe814b07b34ec6b3c451c77d" member_type: VOTER last_known_addr { host: "127.8.218.126" port: 39369 } } peers { permanent_uuid: "88020bf871104cab8c5dd634c3f6f55d" member_type: VOTER last_known_addr { host: "127.8.218.125" port: 36841 } } peers { permanent_uuid: "24ef2e832f624a8cb3741d753b948b9f" member_type: VOTER last_known_addr { host: "127.8.218.124" port: 43757 } }
08:07:54.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.687397 9145 raft_consensus.cc:385] T 00000000000000000000000000000000 P 4afce85cbe814b07b34ec6b3c451c77d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:07:54.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.688522 9145 raft_consensus.cc:740] T 00000000000000000000000000000000 P 4afce85cbe814b07b34ec6b3c451c77d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4afce85cbe814b07b34ec6b3c451c77d, State: Initialized, Role: FOLLOWER
08:07:54.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.690986 9145 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4afce85cbe814b07b34ec6b3c451c77d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4afce85cbe814b07b34ec6b3c451c77d" member_type: VOTER last_known_addr { host: "127.8.218.126" port: 39369 } } peers { permanent_uuid: "88020bf871104cab8c5dd634c3f6f55d" member_type: VOTER last_known_addr { host: "127.8.218.125" port: 36841 } } peers { permanent_uuid: "24ef2e832f624a8cb3741d753b948b9f" member_type: VOTER last_known_addr { host: "127.8.218.124" port: 43757 } }
08:07:54.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.715021 9120 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "24ef2e832f624a8cb3741d753b948b9f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4afce85cbe814b07b34ec6b3c451c77d" is_pre_election: true
08:07:54.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.715866 9120 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 4afce85cbe814b07b34ec6b3c451c77d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 24ef2e832f624a8cb3741d753b948b9f in term 0.
08:07:54.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.716815 9119 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "24ef2e832f624a8cb3741d753b948b9f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4afce85cbe814b07b34ec6b3c451c77d"
08:07:54.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.717360 9119 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 4afce85cbe814b07b34ec6b3c451c77d [term 0 FOLLOWER]: Advancing to term 1
08:07:54.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.727705 9119 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 4afce85cbe814b07b34ec6b3c451c77d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 24ef2e832f624a8cb3741d753b948b9f in term 1.
08:07:54.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.729360 9118 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 4afce85cbe814b07b34ec6b3c451c77d [term 1 FOLLOWER]: Refusing update from remote peer 24ef2e832f624a8cb3741d753b948b9f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:07:54.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:54.732779 9099 tablet.cc:2404] T 00000000000000000000000000000000 P 4afce85cbe814b07b34ec6b3c451c77d: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:07:54.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.736956 9331 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 24ef2e832f624a8cb3741d753b948b9f [LEADER]: Connected to new peer: Peer: permanent_uuid: "4afce85cbe814b07b34ec6b3c451c77d" member_type: VOTER last_known_addr { host: "127.8.218.126" port: 39369 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:07:54.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.747795 9350 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4afce85cbe814b07b34ec6b3c451c77d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4afce85cbe814b07b34ec6b3c451c77d" member_type: VOTER last_known_addr { host: "127.8.218.126" port: 39369 } } peers { permanent_uuid: "88020bf871104cab8c5dd634c3f6f55d" member_type: VOTER last_known_addr { host: "127.8.218.125" port: 36841 } } peers { permanent_uuid: "24ef2e832f624a8cb3741d753b948b9f" member_type: VOTER last_known_addr { host: "127.8.218.124" port: 43757 } } }
08:07:54.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.754684 9350 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4afce85cbe814b07b34ec6b3c451c77d [sys.catalog]: This master's current role is: FOLLOWER
08:07:54.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.748972 9145 sys_catalog.cc:565] T 00000000000000000000000000000000 P 4afce85cbe814b07b34ec6b3c451c77d [sys.catalog]: configured and running, proceeding with master startup.
08:07:54.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.800683 9352 mvcc.cc:204] Tried to move back new op lower bound from 7281851081320910848 to 7281851081051357184. Current Snapshot: MvccSnapshot[applied={T|T < 7281851081320910848}]
08:07:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.829183 9351 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4afce85cbe814b07b34ec6b3c451c77d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 24ef2e832f624a8cb3741d753b948b9f. Latest consensus state: current_term: 1 leader_uuid: "24ef2e832f624a8cb3741d753b948b9f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4afce85cbe814b07b34ec6b3c451c77d" member_type: VOTER last_known_addr { host: "127.8.218.126" port: 39369 } } peers { permanent_uuid: "88020bf871104cab8c5dd634c3f6f55d" member_type: VOTER last_known_addr { host: "127.8.218.125" port: 36841 } } peers { permanent_uuid: "24ef2e832f624a8cb3741d753b948b9f" member_type: VOTER last_known_addr { host: "127.8.218.124" port: 43757 } } }
08:07:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.830222 9351 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4afce85cbe814b07b34ec6b3c451c77d [sys.catalog]: This master's current role is: FOLLOWER
08:07:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.829893 9350 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4afce85cbe814b07b34ec6b3c451c77d [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "24ef2e832f624a8cb3741d753b948b9f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4afce85cbe814b07b34ec6b3c451c77d" member_type: VOTER last_known_addr { host: "127.8.218.126" port: 39369 } } peers { permanent_uuid: "88020bf871104cab8c5dd634c3f6f55d" member_type: VOTER last_known_addr { host: "127.8.218.125" port: 36841 } } peers { permanent_uuid: "24ef2e832f624a8cb3741d753b948b9f" member_type: VOTER last_known_addr { host: "127.8.218.124" port: 43757 } } }
08:07:54.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.830694 9350 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4afce85cbe814b07b34ec6b3c451c77d [sys.catalog]: This master's current role is: FOLLOWER
08:07:54.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:54.867740 9369 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
08:07:54.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:54.925982 9346 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:07:54.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:54.926661 9346 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:07:54.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:54.927206 9346 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:07:54.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:54.963860 9346 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:07:54.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:54.964748 9346 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.218.65
08:07:55.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:55.003520 9346 tablet_server_runner.cc:78] Tablet server non-default flags:
08:07:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.218.84:41765
08:07:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-0/data
08:07:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-0/wal
08:07:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.218.65:0
08:07:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:07:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-0/data/info.pb
08:07:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.218.65
08:07:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.218.126:39369,127.8.218.125:36841,127.8.218.124:43757
08:07:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:07:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:07:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:07:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.218.65
08:07:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-0/logs
08:07:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:07:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:07:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:07:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:07:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
08:07:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:07:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 07:43:21 UTC on e7f111948823
08:07:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11723
08:07:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:07:55.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:55.005183 9346 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:07:55.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:55.007078 9346 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:07:55.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:55.021174 9372 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:55.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:55.126709 9338 catalog_manager.cc:1269] Loaded cluster ID: 4bfe83002d3344f9b16c27f2d2fedbee
08:07:55.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:55.127231 9338 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 88020bf871104cab8c5dd634c3f6f55d: loading cluster ID for follower catalog manager: success
08:07:55.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:55.139855 9338 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 88020bf871104cab8c5dd634c3f6f55d: acquiring CA information for follower catalog manager: success
08:07:55.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:55.154893 9338 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 88020bf871104cab8c5dd634c3f6f55d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:07:55.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:55.168428 9331 sys_catalog.cc:455] T 00000000000000000000000000000000 P 24ef2e832f624a8cb3741d753b948b9f [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "24ef2e832f624a8cb3741d753b948b9f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4afce85cbe814b07b34ec6b3c451c77d" member_type: VOTER last_known_addr { host: "127.8.218.126" port: 39369 } } peers { permanent_uuid: "88020bf871104cab8c5dd634c3f6f55d" member_type: VOTER last_known_addr { host: "127.8.218.125" port: 36841 } } peers { permanent_uuid: "24ef2e832f624a8cb3741d753b948b9f" member_type: VOTER last_known_addr { host: "127.8.218.124" port: 43757 } } }
08:07:55.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:55.170481 9331 sys_catalog.cc:458] T 00000000000000000000000000000000 P 24ef2e832f624a8cb3741d753b948b9f [sys.catalog]: This master's current role is: LEADER
08:07:55.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:55.902302 9369 catalog_manager.cc:1269] Loaded cluster ID: 4bfe83002d3344f9b16c27f2d2fedbee
08:07:55.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:55.912132 9369 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 4afce85cbe814b07b34ec6b3c451c77d: loading cluster ID for follower catalog manager: success
08:07:55.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:55.932855 9369 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 4afce85cbe814b07b34ec6b3c451c77d: acquiring CA information for follower catalog manager: success
08:07:55.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:55.955694 9369 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 4afce85cbe814b07b34ec6b3c451c77d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:07:56.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:56.422726 9371 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 9346
08:07:56.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:56.723620 9374 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1698 milliseconds
08:07:56.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:56.723968 9346 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.688s user 0.000s sys 0.002s
08:07:56.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:56.725292 9346 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.690s user 0.000s sys 0.003s
08:07:56.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:55.024346 9373 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:56.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:56.727483 9346 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:07:56.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:56.728192 9376 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:56.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:56.729862 9346 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:07:56.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:56.732538 9346 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:07:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:56.734098 9346 hybrid_clock.cc:648] HybridClock initialized: now 1777795676734019 us; error 88 us; skew 500 ppm
08:07:56.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:56.735198 9346 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:07:56.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:56.750751 9346 webserver.cc:492] Webserver started at http://127.8.218.65:37513/ using document root <none> and password file <none>
08:07:56.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:56.751756 9346 fs_manager.cc:362] Metadata directory not provided
08:07:56.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:56.752075 9346 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:07:56.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:56.752614 9346 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:07:56.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:56.757133 9346 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-0/data/instance:
08:07:56.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eb4b8c11b07c4697ab4b4a7e1bd0aef7"
08:07:56.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 08:07:56 on dist-test-slave-tr3r"
08:07:56.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:56.758317 9346 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-0/wal/instance:
08:07:56.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eb4b8c11b07c4697ab4b4a7e1bd0aef7"
08:07:56.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 08:07:56 on dist-test-slave-tr3r"
08:07:56.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:56.766204 9346 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
08:07:56.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:56.772259 9386 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:07:56.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:56.773439 9346 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
08:07:56.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:56.773820 9346 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-0/data,/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-0/wal
08:07:56.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eb4b8c11b07c4697ab4b4a7e1bd0aef7"
08:07:56.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 08:07:56 on dist-test-slave-tr3r"
08:07:56.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:56.774187 9346 fs_report.cc:389] FS layout report
08:07:56.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:07:56.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-0/wal
08:07:56.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-0/wal
08:07:56.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-0/data/data
08:07:56.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:07:56.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:07:56.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:07:56.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:07:56.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:07:56.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:07:56.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:07:56.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:07:56.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:07:56.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:07:56.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:07:56.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:56.843860 9346 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:07:56.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:56.845409 9346 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:07:56.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:56.845953 9346 kserver.cc:163] Server-wide thread pool size limit: 3276
08:07:56.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:56.848982 9346 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:07:56.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:56.853992 9346 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:07:56.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:56.854226 9346 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:07:56.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:56.854525 9346 ts_tablet_manager.cc:616] Registered 0 tablets
08:07:56.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:56.854697 9346 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:07:57.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:57.033667 9346 rpc_server.cc:307] RPC server started. Bound to: 127.8.218.65:38843
08:07:57.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:57.033823 9498 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.218.65:38843 every 8 connection(s)
08:07:57.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:57.037521 9346 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-0/data/info.pb
08:07:57.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:57.039933 9065 external_mini_cluster.cc:1428] Started /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu as pid 9346
08:07:57.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:57.040429 9065 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-0/wal/instance
08:07:57.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:57.053732 9065 external_mini_cluster.cc:1366] Running /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu
08:07:57.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu
08:07:57.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-1/wal
08:07:57.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-1/data
08:07:57.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:07:57.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:07:57.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:07:57.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:57.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:57.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:57.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-1/logs
08:07:57.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-1/data/info.pb
08:07:57.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:57.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:07:57.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:07:57.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:07:57.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:07:57.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:57.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:57.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:57.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:07:57.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:07:57.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.218.66:0
08:07:57.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.218.66
08:07:57.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.218.66
08:07:57.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:57.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.218.126:39369,127.8.218.125:36841,127.8.218.124:43757
08:07:57.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.218.84:41765
08:07:57.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:57.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:57.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:07:57.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:57.108373 9499 heartbeater.cc:344] Connected to a master server at 127.8.218.124:43757
08:07:57.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:57.108927 9499 heartbeater.cc:461] Registering TS with master...
08:07:57.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:57.110373 9499 heartbeater.cc:507] Master 127.8.218.124:43757 requested a full tablet report, sending...
08:07:57.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:57.113933 9266 ts_manager.cc:194] Registered new tserver with Master: eb4b8c11b07c4697ab4b4a7e1bd0aef7 (127.8.218.65:38843)
08:07:57.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:57.116600 9266 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.8.218.65:34149
08:07:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:57.116396 9503 heartbeater.cc:344] Connected to a master server at 127.8.218.126:39369
08:07:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:57.122537 9503 heartbeater.cc:461] Registering TS with master...
08:07:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:57.118299 9500 heartbeater.cc:344] Connected to a master server at 127.8.218.125:36841
08:07:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:57.123664 9500 heartbeater.cc:461] Registering TS with master...
08:07:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:57.124289 9500 heartbeater.cc:507] Master 127.8.218.125:36841 requested a full tablet report, sending...
08:07:57.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:57.129204 9185 ts_manager.cc:194] Registered new tserver with Master: eb4b8c11b07c4697ab4b4a7e1bd0aef7 (127.8.218.65:38843)
08:07:57.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:57.123271 9503 heartbeater.cc:507] Master 127.8.218.126:39369 requested a full tablet report, sending...
08:07:57.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:57.134831 9110 ts_manager.cc:194] Registered new tserver with Master: eb4b8c11b07c4697ab4b4a7e1bd0aef7 (127.8.218.65:38843)
08:07:57.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:57.503962 9507 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:07:57.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:57.504527 9507 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:07:57.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:57.505074 9507 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:07:57.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:57.542054 9507 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:07:57.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:57.542941 9507 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.218.66
08:07:57.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:57.581045 9507 tablet_server_runner.cc:78] Tablet server non-default flags:
08:07:57.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:57.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.218.84:41765
08:07:57.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:57.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:57.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-1/data
08:07:57.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-1/wal
08:07:57.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:57.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:57.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.218.66:0
08:07:57.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:07:57.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:57.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:57.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-1/data/info.pb
08:07:57.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.218.66
08:07:57.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:57.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.218.126:39369,127.8.218.125:36841,127.8.218.124:43757
08:07:57.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:07:57.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:57.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:07:57.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:07:57.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:57.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.218.66
08:07:57.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-1/logs
08:07:57.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:57.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:07:57.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:07:57.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:07:57.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:07:57.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
08:07:57.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:07:57.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 07:43:21 UTC on e7f111948823
08:07:57.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11723
08:07:57.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:07:57.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:57.582625 9507 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:07:57.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:57.584245 9507 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:07:57.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:57.596863 9515 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:58.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:58.122285 9499 heartbeater.cc:499] Master 127.8.218.124:43757 was elected leader, sending a full tablet report...
08:07:59.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:58.998337 9514 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 9507
08:07:59.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:59.223770 9514 kernel_stack_watchdog.cc:198] Thread 9507 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 397ms:
08:07:59.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:07:59.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:07:59.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:07:59.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:07:59.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:57.598224 9516 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:59.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:59.224740 9507 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.626s user 0.000s sys 0.001s
08:07:59.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:59.225091 9507 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.626s user 0.000s sys 0.002s
08:07:59.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:59.226981 9519 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:59.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:59.229945 9517 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1625 milliseconds
08:07:59.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:59.229970 9507 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:07:59.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:59.231359 9507 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:07:59.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:59.233860 9507 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:07:59.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:59.235318 9507 hybrid_clock.cc:648] HybridClock initialized: now 1777795679235278 us; error 54 us; skew 500 ppm
08:07:59.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:59.236099 9507 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:07:59.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:59.241997 9507 webserver.cc:492] Webserver started at http://127.8.218.66:35691/ using document root <none> and password file <none>
08:07:59.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:59.242877 9507 fs_manager.cc:362] Metadata directory not provided
08:07:59.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:59.243058 9507 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:07:59.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:59.243453 9507 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:07:59.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:59.247642 9507 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-1/data/instance:
08:07:59.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c4efe6433d894a7ca2b14a61c40e9505"
08:07:59.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 08:07:59 on dist-test-slave-tr3r"
08:07:59.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:59.248744 9507 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-1/wal/instance:
08:07:59.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c4efe6433d894a7ca2b14a61c40e9505"
08:07:59.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 08:07:59 on dist-test-slave-tr3r"
08:07:59.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:59.255906 9507 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.003s
08:07:59.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:59.261369 9527 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:07:59.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:59.262566 9507 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
08:07:59.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:59.262863 9507 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-1/data,/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-1/wal
08:07:59.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c4efe6433d894a7ca2b14a61c40e9505"
08:07:59.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 08:07:59 on dist-test-slave-tr3r"
08:07:59.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:59.263165 9507 fs_report.cc:389] FS layout report
08:07:59.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:07:59.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-1/wal
08:07:59.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-1/wal
08:07:59.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-1/data/data
08:07:59.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:07:59.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:07:59.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:07:59.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:07:59.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:07:59.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:07:59.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:07:59.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:07:59.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:07:59.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:07:59.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:07:59.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:59.323705 9507 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:07:59.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:59.325227 9507 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:07:59.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:59.325635 9507 kserver.cc:163] Server-wide thread pool size limit: 3276
08:07:59.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:59.328226 9507 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:07:59.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:59.332585 9507 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:07:59.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:59.332808 9507 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:07:59.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:59.333010 9507 ts_tablet_manager.cc:616] Registered 0 tablets
08:07:59.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:59.333143 9507 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:07:59.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:59.464790 9507 rpc_server.cc:307] RPC server started. Bound to: 127.8.218.66:42625
08:07:59.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:59.464906 9639 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.218.66:42625 every 8 connection(s)
08:07:59.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:59.467315 9507 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-1/data/info.pb
08:07:59.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:59.469866 9065 external_mini_cluster.cc:1428] Started /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu as pid 9507
08:07:59.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:59.470618 9065 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-1/wal/instance
08:07:59.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:59.501294 9065 external_mini_cluster.cc:1366] Running /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu
08:07:59.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu
08:07:59.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-2/wal
08:07:59.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-2/data
08:07:59.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:07:59.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:07:59.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:07:59.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:59.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:59.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:59.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-2/logs
08:07:59.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-2/data/info.pb
08:07:59.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:59.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:07:59.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:07:59.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:07:59.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:07:59.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:59.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:59.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:59.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:07:59.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:07:59.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.218.67:0
08:07:59.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.218.67
08:07:59.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.218.67
08:07:59.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:59.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.218.126:39369,127.8.218.125:36841,127.8.218.124:43757
08:07:59.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.218.84:41765
08:07:59.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:59.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:59.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:07:59.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:59.524081 9641 heartbeater.cc:344] Connected to a master server at 127.8.218.125:36841
08:07:59.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:59.524652 9641 heartbeater.cc:461] Registering TS with master...
08:07:59.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:59.526098 9641 heartbeater.cc:507] Master 127.8.218.125:36841 requested a full tablet report, sending...
08:07:59.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:59.526654 9640 heartbeater.cc:344] Connected to a master server at 127.8.218.124:43757
08:07:59.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:59.527011 9640 heartbeater.cc:461] Registering TS with master...
08:07:59.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:59.527889 9640 heartbeater.cc:507] Master 127.8.218.124:43757 requested a full tablet report, sending...
08:07:59.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:59.529425 9185 ts_manager.cc:194] Registered new tserver with Master: c4efe6433d894a7ca2b14a61c40e9505 (127.8.218.66:42625)
08:07:59.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:59.530373 9266 ts_manager.cc:194] Registered new tserver with Master: c4efe6433d894a7ca2b14a61c40e9505 (127.8.218.66:42625)
08:07:59.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:59.532051 9266 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.8.218.66:42393
08:07:59.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:59.533993 9642 heartbeater.cc:344] Connected to a master server at 127.8.218.126:39369
08:07:59.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:59.534363 9642 heartbeater.cc:461] Registering TS with master...
08:07:59.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:59.535032 9642 heartbeater.cc:507] Master 127.8.218.126:39369 requested a full tablet report, sending...
08:07:59.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:59.537708 9110 ts_manager.cc:194] Registered new tserver with Master: c4efe6433d894a7ca2b14a61c40e9505 (127.8.218.66:42625)
08:07:59.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:59.849655 9650 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:07:59.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:59.850185 9650 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:07:59.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:59.850617 9650 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:07:59.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:59.883971 9650 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:07:59.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:59.884804 9650 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.218.67
08:07:59.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:59.922871 9650 tablet_server_runner.cc:78] Tablet server non-default flags:
08:07:59.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:59.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.218.84:41765
08:07:59.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:59.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:59.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-2/data
08:07:59.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-2/wal
08:07:59.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:59.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:59.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.218.67:0
08:07:59.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:07:59.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:59.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:59.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-2/data/info.pb
08:07:59.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.218.67
08:07:59.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:59.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.218.126:39369,127.8.218.125:36841,127.8.218.124:43757
08:07:59.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:07:59.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:59.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:07:59.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:07:59.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:59.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.218.67
08:07:59.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-2/logs
08:07:59.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:59.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:07:59.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:07:59.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:07:59.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:07:59.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
08:07:59.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:07:59.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 07:43:21 UTC on e7f111948823
08:07:59.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11723
08:07:59.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:07:59.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:59.924175 9650 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:07:59.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:07:59.925895 9650 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:07:59.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:59.938876 9656 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:00.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:00.535764 9640 heartbeater.cc:499] Master 127.8.218.124:43757 was elected leader, sending a full tablet report...
08:08:01.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:07:59.939791 9657 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:01.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:01.321379 9658 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1374 milliseconds
08:08:01.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:01.321981 9659 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:01.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:01.324036 9650 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.385s user 0.000s sys 0.009s
08:08:01.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:01.324301 9650 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.386s user 0.000s sys 0.009s
08:08:01.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:01.324559 9650 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:08:01.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:01.325647 9650 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:08:01.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:01.328114 9650 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:08:01.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:01.329545 9650 hybrid_clock.cc:648] HybridClock initialized: now 1777795681329472 us; error 70 us; skew 500 ppm
08:08:01.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:01.330476 9650 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:08:01.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:01.337906 9650 webserver.cc:492] Webserver started at http://127.8.218.67:43869/ using document root <none> and password file <none>
08:08:01.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:01.338867 9650 fs_manager.cc:362] Metadata directory not provided
08:08:01.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:01.339061 9650 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:08:01.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:01.339457 9650 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:08:01.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:01.343982 9650 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-2/data/instance:
08:08:01.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "25469524ac784f33a961b3ca6bdf80b6"
08:08:01.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 08:08:01 on dist-test-slave-tr3r"
08:08:01.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:01.345060 9650 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-2/wal/instance:
08:08:01.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "25469524ac784f33a961b3ca6bdf80b6"
08:08:01.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 08:08:01 on dist-test-slave-tr3r"
08:08:01.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:01.353461 9650 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.010s sys 0.000s
08:08:01.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:01.360162 9667 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:08:01.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:01.361395 9650 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
08:08:01.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:01.361706 9650 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-2/data,/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-2/wal
08:08:01.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "25469524ac784f33a961b3ca6bdf80b6"
08:08:01.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 08:08:01 on dist-test-slave-tr3r"
08:08:01.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:01.362113 9650 fs_report.cc:389] FS layout report
08:08:01.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:08:01.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-2/wal
08:08:01.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-2/wal
08:08:01.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-2/data/data
08:08:01.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:08:01.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:08:01.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:08:01.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:08:01.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:08:01.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:08:01.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:08:01.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:08:01.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:08:01.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:08:01.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:08:01.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:01.436872 9650 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:08:01.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:01.438414 9650 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:08:01.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:01.438840 9650 kserver.cc:163] Server-wide thread pool size limit: 3276
08:08:01.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:01.441466 9650 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:08:01.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:01.445585 9650 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:08:01.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:01.445828 9650 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.001s sys 0.000s
08:08:01.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:01.446050 9650 ts_tablet_manager.cc:616] Registered 0 tablets
08:08:01.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:01.446183 9650 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:08:01.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:01.589179 9650 rpc_server.cc:307] RPC server started. Bound to: 127.8.218.67:44419
08:08:01.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:01.589346 9779 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.218.67:44419 every 8 connection(s)
08:08:01.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:01.591639 9650 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-2/data/info.pb
08:08:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:01.593351 9065 external_mini_cluster.cc:1428] Started /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu as pid 9650
08:08:01.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:01.593875 9065 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster13153958806173933168/ts-2/wal/instance
08:08:01.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:01.646646 9780 heartbeater.cc:344] Connected to a master server at 127.8.218.124:43757
08:08:01.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:01.647207 9780 heartbeater.cc:461] Registering TS with master...
08:08:01.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:01.648489 9780 heartbeater.cc:507] Master 127.8.218.124:43757 requested a full tablet report, sending...
08:08:01.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:01.651235 9266 ts_manager.cc:194] Registered new tserver with Master: 25469524ac784f33a961b3ca6bdf80b6 (127.8.218.67:44419)
08:08:01.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:01.653043 9266 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.8.218.67:44421
08:08:01.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:01.654830 9781 heartbeater.cc:344] Connected to a master server at 127.8.218.125:36841
08:08:01.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:01.655212 9781 heartbeater.cc:461] Registering TS with master...
08:08:01.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:01.656193 9781 heartbeater.cc:507] Master 127.8.218.125:36841 requested a full tablet report, sending...
08:08:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:01.659003 9185 ts_manager.cc:194] Registered new tserver with Master: 25469524ac784f33a961b3ca6bdf80b6 (127.8.218.67:44419)
08:08:01.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:01.660614 9782 heartbeater.cc:344] Connected to a master server at 127.8.218.126:39369
08:08:01.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:01.660930 9782 heartbeater.cc:461] Registering TS with master...
08:08:01.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:01.661553 9782 heartbeater.cc:507] Master 127.8.218.126:39369 requested a full tablet report, sending...
08:08:01.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:01.663890 9110 ts_manager.cc:194] Registered new tserver with Master: 25469524ac784f33a961b3ca6bdf80b6 (127.8.218.67:44419)
08:08:01.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:01.668972 9065 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
08:08:01.679 [DEBUG - main] (MiniKuduCluster.java:195) Response:
08:08:01.679 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
08:08:01.680 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.8.218.126"
port: 39369
}
bound_http_address {
host: "127.8.218.126"
port: 40891
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.8.218.125"
port: 36841
}
bound_http_address {
host: "127.8.218.125"
port: 39007
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.8.218.124"
port: 43757
}
bound_http_address {
host: "127.8.218.124"
port: 35007
}
}
}
08:08:01.681 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
08:08:01.682 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.8.218.65"
port: 38843
}
bound_http_address {
host: "127.8.218.65"
port: 37513
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.8.218.66"
port: 42625
}
bound_http_address {
host: "127.8.218.66"
port: 35691
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.8.218.67"
port: 44419
}
bound_http_address {
host: "127.8.218.67"
port: 43869
}
}
}
08:08:01.682 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
08:08:01.683 [DEBUG - main] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster2267375709918048441/krb5kdc/krb5cc
08:08:01.683 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
08:08:01.684 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task_nU1WP/test-tmp
08:08:01.684 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
08:08:01.685 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task_nU1WP/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
08:08:01.687 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
08:08:02.005 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:08:02.005 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
08:08:02.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:02.008456 9791 mini_oidc.cc:150] Starting JWKS server
08:08:02.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:02.011667 9791 webserver.cc:492] Webserver started at http://127.0.0.1:36329/ using document root <none> and password file <none>
08:08:02.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:02.012092 9791 mini_oidc.cc:186] Starting OIDC Discovery server
08:08:02.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:02.013888 9791 webserver.cc:492] Webserver started at http://127.0.0.1:40701/ using document root <none> and password file <none>
08:08:02.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
08:08:02.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/principal' for realm 'KRBTEST.COM',
08:08:02.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
08:08:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:08:02 dist-test-slave-tr3r krb5kdc[9803](info): setting up network...
08:08:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
08:08:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:08:02 dist-test-slave-tr3r krb5kdc[9803](info): set up 2 sockets
08:08:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:08:02 dist-test-slave-tr3r krb5kdc[9803](info): commencing operation
08:08:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
08:08:02.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:02.657845 9780 heartbeater.cc:499] Master 127.8.218.124:43757 was elected leader, sending a full tablet report...
08:08:03.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:03.969177 9791 mini_kdc.cc:121] Time spent starting KDC: real 1.916s user 0.000s sys 0.019s
08:08:03.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
08:08:03.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:08:03.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
08:08:04.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
08:08:04.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:08:04.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
08:08:04.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
08:08:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:08:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
08:08:04.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:08:04.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/test-user.keytab.
08:08:04.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/test-user.keytab.
08:08:04.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:08:04 dist-test-slave-tr3r krb5kdc[9803](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777795684, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:08:04.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
08:08:04.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-03T08:08:04Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
08:08:04.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-03T08:08:04Z Disabled control of system clock
08:08:04.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.9.143.254@KRBTEST.COM; defaulting to no policy
08:08:04.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:08:04.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.9.143.254@KRBTEST.COM" created.
08:08:04.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:08:04.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.143.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kudu.keytab.
08:08:04.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.143.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kudu.keytab.
08:08:04.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.9.143.254@KRBTEST.COM; defaulting to no policy
08:08:04.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:08:04.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.9.143.254@KRBTEST.COM" created.
08:08:04.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:08:04.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.9.143.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kudu.keytab.
08:08:04.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.9.143.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kudu.keytab.
08:08:04.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:04.192446 9791 external_mini_cluster.cc:1366] Running /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu
08:08:04.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu
08:08:04.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/wal
08:08:04.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/data
08:08:04.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:08:04.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:08:04.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:08:04.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:04.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:04.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:04.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/logs
08:08:04.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/data/info.pb
08:08:04.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:04.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:08:04.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:08:04.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:08:04.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:08:04.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:04.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:04.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:04.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:08:04.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:08:04.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:08:04.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:08:04.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.143.254:42279
08:08:04.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.143.254
08:08:04.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:08:04.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kudu.keytab
08:08:04.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.143.254
08:08:04.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:08:04.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:08:04.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:08:04.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:08:04.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.143.212:44461
08:08:04.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:04.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:04.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:04.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:08:04.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.143.254:42279,127.9.143.253:32923,127.9.143.252:37215
08:08:04.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:08:04.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36329/jwks.json
08:08:04.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:08:04.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:08:04.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:04.490006 9821 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:08:04.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:04.490561 9821 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:08:04.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:04.491104 9821 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:08:04.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:04.523584 9821 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
08:08:04.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:04.523845 9821 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:08:04.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:04.524237 9821 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:08:04.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:04.524542 9821 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:08:04.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:04.524840 9821 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:08:04.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:04.525130 9821 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:08:04.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:04.525372 9821 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:36329/jwks.json
08:08:04.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:04.565532 9821 master_runner.cc:386] Master server non-default flags:
08:08:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.143.212:44461
08:08:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/data
08:08:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/wal
08:08:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:08:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:08:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.143.254:42279,127.9.143.253:32923,127.9.143.252:37215
08:08:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kudu.keytab
08:08:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.143.254
08:08:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:08:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.143.254:42279
08:08:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:08:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:08:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:08:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36329/jwks.json
08:08:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:08:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:08:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:08:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/data/info.pb
08:08:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.143.254
08:08:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:08:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:08:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:08:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:08:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/logs
08:08:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:08:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:08:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:08:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:08:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
08:08:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:08:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 07:43:21 UTC on e7f111948823
08:08:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11723
08:08:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:08:04.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:04.567096 9821 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:08:04.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:04.568181 9821 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:08:04.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:04.578629 9827 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:05.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:04.580112 9828 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:05.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:04.582991 9830 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:05.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:05.781651 9821 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:08:05.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:05.781904 9829 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1200 milliseconds
08:08:05.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:05.787551 9821 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:08:05.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:05.790885 9821 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:08:05.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:05.792544 9821 hybrid_clock.cc:648] HybridClock initialized: now 1777795685792467 us; error 83 us; skew 500 ppm
08:08:05.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:05.793610 9821 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:08:05.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:08:05 dist-test-slave-tr3r krb5kdc[9803](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777795685, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.143.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:08:05.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:05.804768 9821 init.cc:377] Logged in from keytab as kudu/127.9.143.254@KRBTEST.COM (short username kudu)
08:08:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:05.810407 9821 webserver.cc:492] Webserver started at http://127.9.143.254:42699/ using document root <none> and password file <none>
08:08:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:05.811376 9821 fs_manager.cc:362] Metadata directory not provided
08:08:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:05.811587 9821 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:08:05.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:05.812021 9821 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:08:05.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:05.816893 9821 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/data/instance:
08:08:05.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "513a338fca7944dbbf4433e106c2a03f"
08:08:05.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 08:08:05 on dist-test-slave-tr3r"
08:08:05.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:05.818300 9821 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/wal/instance:
08:08:05.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "513a338fca7944dbbf4433e106c2a03f"
08:08:05.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 08:08:05 on dist-test-slave-tr3r"
08:08:05.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:05.826594 9821 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.006s sys 0.001s
08:08:05.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:05.832566 9840 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:08:05.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:05.833843 9821 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
08:08:05.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:05.834160 9821 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/data,/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/wal
08:08:05.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "513a338fca7944dbbf4433e106c2a03f"
08:08:05.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 08:08:05 on dist-test-slave-tr3r"
08:08:05.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:05.834481 9821 fs_report.cc:389] FS layout report
08:08:05.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:08:05.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/wal
08:08:05.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/wal
08:08:05.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/data/data
08:08:05.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:08:05.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:08:05.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:08:05.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:08:05.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:08:05.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:08:05.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:08:05.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:08:05.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:08:05.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:08:05.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:08:05.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:05.884712 9821 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:08:05.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:05.894707 9821 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:08:05.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:05.895321 9821 kserver.cc:163] Server-wide thread pool size limit: 3276
08:08:05.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:05.977098 9821 rpc_server.cc:307] RPC server started. Bound to: 127.9.143.254:42279
08:08:05.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:05.977216 9891 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.143.254:42279 every 8 connection(s)
08:08:05.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:05.979851 9821 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/data/info.pb
08:08:05.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:05.982125 9791 external_mini_cluster.cc:1428] Started /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu as pid 9821
08:08:05.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:05.982685 9791 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/wal/instance
08:08:05.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:05.986565 9892 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:08:05.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.9.143.253@KRBTEST.COM; defaulting to no policy
08:08:05.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:05.997272 9892 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } has no permanent_uuid. Determining permanent_uuid...
08:08:06.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:08:06.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.9.143.253@KRBTEST.COM" created.
08:08:06.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:08:06 dist-test-slave-tr3r krb5kdc[9803](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777795685, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.143.254@KRBTEST.COM for kudu/127.9.143.254@KRBTEST.COM
08:08:06.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:08:06.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.143.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kudu.keytab.
08:08:06.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.143.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kudu.keytab.
08:08:06.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:06.036420 9892 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } has no permanent_uuid. Determining permanent_uuid...
08:08:06.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:06.038906 9842 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.143.253:32923: connect: Connection refused (error 111)
08:08:06.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.9.143.253@KRBTEST.COM; defaulting to no policy
08:08:06.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:08:06.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.9.143.253@KRBTEST.COM" created.
08:08:06.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:06.047410 9892 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.143.253:32923: Network error: Client connection negotiation failed: client connection to 127.9.143.253:32923: connect: Connection refused (error 111)
08:08:06.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:08:06.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.9.143.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kudu.keytab.
08:08:06.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.9.143.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kudu.keytab.
08:08:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:06.071151 9791 external_mini_cluster.cc:1366] Running /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu
08:08:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu
08:08:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/wal
08:08:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/data
08:08:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:08:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:08:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:08:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/logs
08:08:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/data/info.pb
08:08:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:08:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:08:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:08:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:08:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:08:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:08:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:08:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:08:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.143.253:32923
08:08:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.143.253
08:08:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:08:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kudu.keytab
08:08:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.143.253
08:08:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:08:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:08:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:08:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:08:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.143.212:44461
08:08:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:08:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.143.254:42279,127.9.143.253:32923,127.9.143.252:37215
08:08:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:08:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36329/jwks.json
08:08:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:08:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:08:06.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:06.097000 9892 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } attempt: 1
08:08:06.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:06.100909 9892 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.143.253:32923: Network error: Client connection negotiation failed: client connection to 127.9.143.253:32923: connect: Connection refused (error 111)
08:08:06.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:06.169276 9892 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } attempt: 2
08:08:06.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:06.173439 9892 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.143.253:32923: Network error: Client connection negotiation failed: client connection to 127.9.143.253:32923: connect: Connection refused (error 111)
08:08:06.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:06.264928 9892 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } attempt: 3
08:08:06.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:06.269286 9892 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.143.253:32923: Network error: Client connection negotiation failed: client connection to 127.9.143.253:32923: connect: Connection refused (error 111)
08:08:06.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:06.371975 9901 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:08:06.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:06.372579 9901 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:08:06.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:06.373267 9901 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:08:06.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:06.408816 9901 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
08:08:06.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:06.409060 9901 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:08:06.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:06.409390 9901 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:08:06.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:06.409611 9901 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:08:06.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:06.409854 9901 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:08:06.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:06.410060 9901 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:08:06.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:06.410220 9901 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:36329/jwks.json
08:08:06.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:06.412704 9892 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } attempt: 4
08:08:06.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:06.416703 9892 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.143.253:32923: Network error: Client connection negotiation failed: client connection to 127.9.143.253:32923: connect: Connection refused (error 111)
08:08:06.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:06.448920 9901 master_runner.cc:386] Master server non-default flags:
08:08:06.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:06.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.143.212:44461
08:08:06.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:06.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:06.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/data
08:08:06.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/wal
08:08:06.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:08:06.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:08:06.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.143.254:42279,127.9.143.253:32923,127.9.143.252:37215
08:08:06.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kudu.keytab
08:08:06.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.143.253
08:08:06.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:06.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:06.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:08:06.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.143.253:32923
08:08:06.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:08:06.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:08:06.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:08:06.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36329/jwks.json
08:08:06.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:08:06.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:08:06.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:08:06.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:06.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:06.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/data/info.pb
08:08:06.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.143.253
08:08:06.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:08:06.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:08:06.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:06.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:08:06.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:08:06.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:06.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/logs
08:08:06.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:06.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:08:06.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:08:06.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:08:06.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:08:06.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
08:08:06.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:08:06.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 07:43:21 UTC on e7f111948823
08:08:06.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11723
08:08:06.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:08:06.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:06.450330 9901 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:08:06.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:06.451294 9901 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:08:06.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:06.462165 9907 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:06.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:06.716136 9892 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } attempt: 5
08:08:06.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:06.722425 9892 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.143.253:32923: Network error: Client connection negotiation failed: client connection to 127.9.143.253:32923: connect: Connection refused (error 111)
08:08:07.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:07.269906 9892 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } attempt: 6
08:08:07.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:07.299853 9892 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.143.253:32923: Network error: Client connection negotiation failed: client connection to 127.9.143.253:32923: connect: Connection refused (error 111)
08:08:08.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:06.462975 9908 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:08.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:07.865424 9906 debug-util.cc:398] Leaking SignalData structure 0x7b0800028d00 after lost signal to thread 9901
08:08:08.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:08.041672 9906 kernel_stack_watchdog.cc:198] Thread 9901 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 401ms:
08:08:08.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:08:08.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:08:08.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:08:08.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:08:08.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:08.045454 9901 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.583s user 0.002s sys 0.000s
08:08:08.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:08.046032 9909 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1582 milliseconds
08:08:08.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:08.045866 9901 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.584s user 0.002s sys 0.000s
08:08:08.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:08.047822 9910 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:08.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:08.047911 9901 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:08:08.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:08.049118 9901 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:08:08.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:08.051900 9901 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:08:08.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:08.053321 9901 hybrid_clock.cc:648] HybridClock initialized: now 1777795688053279 us; error 37 us; skew 500 ppm
08:08:08.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:08.054286 9901 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:08:08.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:08:08 dist-test-slave-tr3r krb5kdc[9803](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777795688, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.143.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:08:08.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:08.060443 9901 init.cc:377] Logged in from keytab as kudu/127.9.143.253@KRBTEST.COM (short username kudu)
08:08:08.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:08.065621 9901 webserver.cc:492] Webserver started at http://127.9.143.253:37721/ using document root <none> and password file <none>
08:08:08.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:08.066716 9901 fs_manager.cc:362] Metadata directory not provided
08:08:08.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:08.066939 9901 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:08:08.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:08.067505 9901 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:08:08.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:08.072177 9901 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/data/instance:
08:08:08.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1d36c7228de444508830a92c33ddaa4a"
08:08:08.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 08:08:08 on dist-test-slave-tr3r"
08:08:08.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:08.073340 9901 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/wal/instance:
08:08:08.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1d36c7228de444508830a92c33ddaa4a"
08:08:08.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 08:08:08 on dist-test-slave-tr3r"
08:08:08.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:08.080886 9901 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.000s
08:08:08.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:08.086489 9922 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:08:08.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:08.087769 9901 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
08:08:08.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:08.088099 9901 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/data,/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/wal
08:08:08.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1d36c7228de444508830a92c33ddaa4a"
08:08:08.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 08:08:08 on dist-test-slave-tr3r"
08:08:08.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:08.088416 9901 fs_report.cc:389] FS layout report
08:08:08.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:08:08.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/wal
08:08:08.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/wal
08:08:08.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/data/data
08:08:08.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:08:08.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:08:08.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:08:08.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:08:08.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:08:08.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:08:08.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:08:08.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:08:08.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:08:08.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:08:08.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:08:08.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:08.155022 9901 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:08:08.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:08.160988 9901 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:08:08.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:08.161550 9901 kserver.cc:163] Server-wide thread pool size limit: 3276
08:08:08.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:08.233129 9901 rpc_server.cc:307] RPC server started. Bound to: 127.9.143.253:32923
08:08:08.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:08.233188 9973 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.143.253:32923 every 8 connection(s)
08:08:08.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:08.235733 9901 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/data/info.pb
08:08:08.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:08.241559 9974 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:08:08.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:08.243474 9791 external_mini_cluster.cc:1428] Started /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu as pid 9901
08:08:08.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:08.243889 9791 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/wal/instance
08:08:08.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:08.253721 9974 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } has no permanent_uuid. Determining permanent_uuid...
08:08:08.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.9.143.252@KRBTEST.COM; defaulting to no policy
08:08:08.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:08:08.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.9.143.252@KRBTEST.COM" created.
08:08:08.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:08:08 dist-test-slave-tr3r krb5kdc[9803](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777795688, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.143.253@KRBTEST.COM for kudu/127.9.143.254@KRBTEST.COM
08:08:08.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:08:08.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.143.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kudu.keytab.
08:08:08.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.143.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kudu.keytab.
08:08:08.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:08.297150 9974 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } has no permanent_uuid. Determining permanent_uuid...
08:08:08.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.9.143.252@KRBTEST.COM; defaulting to no policy
08:08:08.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:08:08.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.9.143.252@KRBTEST.COM" created.
08:08:08.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:08:08 dist-test-slave-tr3r krb5kdc[9803](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777795688, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.143.253@KRBTEST.COM for kudu/127.9.143.253@KRBTEST.COM
08:08:08.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:08.321635 9974 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } has no permanent_uuid. Determining permanent_uuid...
08:08:08.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:08.324251 9926 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.143.252:37215: connect: Connection refused (error 111)
08:08:08.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:08:08.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.9.143.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kudu.keytab.
08:08:08.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.9.143.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kudu.keytab.
08:08:08.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:08.328329 9974 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.143.252:37215: Network error: Client connection negotiation failed: client connection to 127.9.143.252:37215: connect: Connection refused (error 111)
08:08:08.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:08.330391 9791 external_mini_cluster.cc:1366] Running /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu
08:08:08.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu
08:08:08.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/wal
08:08:08.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/data
08:08:08.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:08:08.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:08:08.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:08:08.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:08.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:08.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:08.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/logs
08:08:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/data/info.pb
08:08:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:08:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:08:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:08:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:08:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:08:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:08:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:08:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:08:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.143.252:37215
08:08:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.143.252
08:08:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:08:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kudu.keytab
08:08:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.143.252
08:08:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:08:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:08:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:08:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:08:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.143.212:44461
08:08:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:08:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.143.254:42279,127.9.143.253:32923,127.9.143.252:37215
08:08:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:08:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36329/jwks.json
08:08:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:08:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:08:08.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:08.360328 9892 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } attempt: 7
08:08:08.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:08:08 dist-test-slave-tr3r krb5kdc[9803](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777795685, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.143.254@KRBTEST.COM for kudu/127.9.143.253@KRBTEST.COM
08:08:08.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:08.377882 9974 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } attempt: 1
08:08:08.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:08.382513 9974 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.143.252:37215: Network error: Client connection negotiation failed: client connection to 127.9.143.252:37215: connect: Connection refused (error 111)
08:08:08.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:08.382910 9892 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } has no permanent_uuid. Determining permanent_uuid...
08:08:08.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:08.393373 9892 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.143.252:37215: Network error: Client connection negotiation failed: client connection to 127.9.143.252:37215: connect: Connection refused (error 111)
08:08:08.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:08.450939 9974 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } attempt: 2
08:08:08.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:08.451794 9892 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } attempt: 1
08:08:08.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:08.455109 9974 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.143.252:37215: Network error: Client connection negotiation failed: client connection to 127.9.143.252:37215: connect: Connection refused (error 111)
08:08:08.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:08.456602 9892 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.143.252:37215: Network error: Client connection negotiation failed: client connection to 127.9.143.252:37215: connect: Connection refused (error 111)
08:08:08.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:08.537964 9892 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } attempt: 2
08:08:08.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:08.542750 9892 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.143.252:37215: Network error: Client connection negotiation failed: client connection to 127.9.143.252:37215: connect: Connection refused (error 111)
08:08:08.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:08.546561 9974 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } attempt: 3
08:08:08.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:08.550912 9974 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.143.252:37215: Network error: Client connection negotiation failed: client connection to 127.9.143.252:37215: connect: Connection refused (error 111)
08:08:08.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:08.628193 9892 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } attempt: 3
08:08:08.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:08.632314 9892 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.143.252:37215: Network error: Client connection negotiation failed: client connection to 127.9.143.252:37215: connect: Connection refused (error 111)
08:08:08.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:08.666664 9984 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:08:08.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:08.667207 9984 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:08:08.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:08.667829 9984 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:08:08.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:08.694291 9974 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } attempt: 4
08:08:08.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:08.698412 9974 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.143.252:37215: Network error: Client connection negotiation failed: client connection to 127.9.143.252:37215: connect: Connection refused (error 111)
08:08:08.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:08.704104 9984 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
08:08:08.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:08.704356 9984 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:08:08.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:08.704638 9984 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:08:08.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:08.704847 9984 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:08:08.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:08.705044 9984 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:08:08.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:08.705240 9984 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:08:08.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:08.705390 9984 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:36329/jwks.json
08:08:08.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:08.744947 9984 master_runner.cc:386] Master server non-default flags:
08:08:08.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:08.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.143.212:44461
08:08:08.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:08.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:08.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/data
08:08:08.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/wal
08:08:08.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:08:08.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:08:08.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.143.254:42279,127.9.143.253:32923,127.9.143.252:37215
08:08:08.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kudu.keytab
08:08:08.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.143.252
08:08:08.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:08.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:08.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:08:08.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.143.252:37215
08:08:08.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:08:08.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:08:08.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:08:08.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36329/jwks.json
08:08:08.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:08:08.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:08:08.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:08:08.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:08.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:08.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/data/info.pb
08:08:08.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.143.252
08:08:08.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:08:08.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:08:08.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:08.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:08:08.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:08:08.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:08.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/logs
08:08:08.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:08.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:08:08.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:08:08.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:08:08.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:08:08.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
08:08:08.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:08:08.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 07:43:21 UTC on e7f111948823
08:08:08.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11723
08:08:08.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:08:08.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:08.746287 9984 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:08:08.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:08.747335 9984 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:08:08.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:08.759011 9994 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:08.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:08.772718 9892 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } attempt: 4
08:08:08.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:08.776508 9892 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.143.252:37215: Network error: Client connection negotiation failed: client connection to 127.9.143.252:37215: connect: Connection refused (error 111)
08:08:08.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:08.997830 9974 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } attempt: 5
08:08:09.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:09.004086 9974 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.143.252:37215: Network error: Client connection negotiation failed: client connection to 127.9.143.252:37215: connect: Connection refused (error 111)
08:08:09.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:09.059926 9892 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } attempt: 5
08:08:09.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:09.065357 9892 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.143.252:37215: Network error: Client connection negotiation failed: client connection to 127.9.143.252:37215: connect: Connection refused (error 111)
08:08:09.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:09.551579 9974 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } attempt: 6
08:08:09.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:09.579032 9974 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.143.252:37215: Network error: Client connection negotiation failed: client connection to 127.9.143.252:37215: connect: Connection refused (error 111)
08:08:09.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:09.617874 9892 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } attempt: 6
08:08:09.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:09.648633 9892 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.143.252:37215: Network error: Client connection negotiation failed: client connection to 127.9.143.252:37215: connect: Connection refused (error 111)
08:08:09.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:08.759758 9995 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:09.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:09.987413 9997 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:09.990559 9996 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1230 milliseconds
08:08:09.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:09.990651 9984 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:08:09.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:09.991890 9984 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:08:09.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:09.994627 9984 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:08:09.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:09.996203 9984 hybrid_clock.cc:648] HybridClock initialized: now 1777795689996124 us; error 71 us; skew 500 ppm
08:08:09.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:09.997044 9984 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:08:10.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:08:09 dist-test-slave-tr3r krb5kdc[9803](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777795689, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.143.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:08:10.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.003549 9984 init.cc:377] Logged in from keytab as kudu/127.9.143.252@KRBTEST.COM (short username kudu)
08:08:10.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.008266 9984 webserver.cc:492] Webserver started at http://127.9.143.252:45863/ using document root <none> and password file <none>
08:08:10.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.009179 9984 fs_manager.cc:362] Metadata directory not provided
08:08:10.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.009383 9984 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:08:10.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.009920 9984 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:08:10.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.014489 9984 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/data/instance:
08:08:10.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ca61a7dcebe145b4a80009b389f47393"
08:08:10.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 08:08:10 on dist-test-slave-tr3r"
08:08:10.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.015702 9984 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/wal/instance:
08:08:10.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ca61a7dcebe145b4a80009b389f47393"
08:08:10.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 08:08:10 on dist-test-slave-tr3r"
08:08:10.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.023470 9984 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.008s sys 0.000s
08:08:10.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.029505 10010 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:08:10.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.030808 9984 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
08:08:10.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.031188 9984 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/data,/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/wal
08:08:10.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ca61a7dcebe145b4a80009b389f47393"
08:08:10.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 08:08:10 on dist-test-slave-tr3r"
08:08:10.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.031626 9984 fs_report.cc:389] FS layout report
08:08:10.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:08:10.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/wal
08:08:10.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/wal
08:08:10.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/data/data
08:08:10.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:08:10.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:08:10.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:08:10.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:08:10.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:08:10.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:08:10.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:08:10.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:08:10.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:08:10.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:08:10.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:08:10.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.085049 9984 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:08:10.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.090583 9984 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:08:10.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.091007 9984 kserver.cc:163] Server-wide thread pool size limit: 3276
08:08:10.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.162073 9984 rpc_server.cc:307] RPC server started. Bound to: 127.9.143.252:37215
08:08:10.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.162144 10062 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.143.252:37215 every 8 connection(s)
08:08:10.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.164686 9984 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/data/info.pb
08:08:10.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.170985 10063 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:08:10.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.172966 9791 external_mini_cluster.cc:1428] Started /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu as pid 9984
08:08:10.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.173360 9791 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/wal/instance
08:08:10.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.184098 10063 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } has no permanent_uuid. Determining permanent_uuid...
08:08:10.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:08:10 dist-test-slave-tr3r krb5kdc[9803](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777795689, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.143.252@KRBTEST.COM for kudu/127.9.143.254@KRBTEST.COM
08:08:10.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:08:10 dist-test-slave-tr3r krb5kdc[9803](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777795684, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.9.143.252@KRBTEST.COM
08:08:10.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.217020 10063 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } has no permanent_uuid. Determining permanent_uuid...
08:08:10.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:08:10 dist-test-slave-tr3r krb5kdc[9803](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777795689, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.143.252@KRBTEST.COM for kudu/127.9.143.253@KRBTEST.COM
08:08:10.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:08:10 dist-test-slave-tr3r krb5kdc[9803](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777795684, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.9.143.253@KRBTEST.COM
08:08:10.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.243335 10063 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } has no permanent_uuid. Determining permanent_uuid...
08:08:10.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:08:10 dist-test-slave-tr3r krb5kdc[9803](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777795689, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.143.252@KRBTEST.COM for kudu/127.9.143.252@KRBTEST.COM
08:08:10.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:08:10 dist-test-slave-tr3r krb5kdc[9803](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777795684, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.9.143.254@KRBTEST.COM
08:08:10.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.271819 10063 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393: Bootstrap starting.
08:08:10.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.274924 9791 external_mini_cluster.cc:949] 0 TS(s) registered with all masters
08:08:10.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.277694 10063 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393: Neither blocks nor log segments found. Creating new log.
08:08:10.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.279546 10063 log.cc:826] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393: Log is configured to *not* fsync() on all Append() calls
08:08:10.283 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:08:10.283 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
08:08:10.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.284320 10063 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393: No bootstrap required, opened a new log
08:08:10.284 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
08:08:10.284 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
08:08:10.285 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.9.143.254"
port: 42279
}
bound_http_address {
host: "127.9.143.254"
port: 42699
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.9.143.253"
port: 32923
}
bound_http_address {
host: "127.9.143.253"
port: 37721
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.9.143.252"
port: 37215
}
bound_http_address {
host: "127.9.143.252"
port: 45863
}
}
}
08:08:10.286 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
08:08:10.286 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:08:10.286 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
08:08:10.286 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
08:08:10.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.304764 10063 raft_consensus.cc:359] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } }
08:08:10.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.305428 10063 raft_consensus.cc:385] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:08:10.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.305629 10063 raft_consensus.cc:740] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ca61a7dcebe145b4a80009b389f47393, State: Initialized, Role: FOLLOWER
08:08:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.306396 10063 consensus_queue.cc:260] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } }
08:08:10.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.309085 10075 sys_catalog.cc:455] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } } }
08:08:10.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.309913 10075 sys_catalog.cc:458] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [sys.catalog]: This master's current role is: FOLLOWER
08:08:10.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.310855 10063 sys_catalog.cc:565] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [sys.catalog]: configured and running, proceeding with master startup.
08:08:10.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:08:10 dist-test-slave-tr3r krb5kdc[9803](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777795690, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:08:10.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
08:08:10.319 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:08:10.321 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/krb5cc
08:08:10.322 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
08:08:10.322 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-user
08:08:10.323 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-user"
}
08:08:10.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:10.346026 10016 tablet.cc:2404] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:08:10.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:10.346309 10087 catalog_manager.cc:1568] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:08:10.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:10.346653 10087 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:08:10.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:08:10 dist-test-slave-tr3r krb5kdc[9803](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777795690, etypes {rep=17 tkt=17 ses=17}, test-user@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:08:10.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-user@KRBTEST.COM:
08:08:10.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.334669 9984 master_runner.cc:186] Error getting master registration for 127.9.143.254:42279: OK, instance_id { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" instance_seqno: 1777795685958862 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:08:10.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.360275 9984 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:08:10.366 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:08:10.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.639596 9974 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } attempt: 7
08:08:10.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:08:10 dist-test-slave-tr3r krb5kdc[9803](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777795688, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.143.253@KRBTEST.COM for kudu/127.9.143.252@KRBTEST.COM
08:08:10.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.671593 9974 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a: Bootstrap starting.
08:08:10.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.679667 9974 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a: Neither blocks nor log segments found. Creating new log.
08:08:10.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.682117 9892 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } attempt: 7
08:08:10.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.682193 9974 log.cc:826] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a: Log is configured to *not* fsync() on all Append() calls
08:08:10.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.689720 9974 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a: No bootstrap required, opened a new log
08:08:10.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.691879 10075 raft_consensus.cc:493] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:08:10.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.692485 10075 raft_consensus.cc:515] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } }
08:08:10.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.696487 10075 leader_election.cc:290] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 513a338fca7944dbbf4433e106c2a03f (127.9.143.254:42279), 1d36c7228de444508830a92c33ddaa4a (127.9.143.253:32923)
08:08:10.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:08:10 dist-test-slave-tr3r krb5kdc[9803](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777795685, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.143.254@KRBTEST.COM for kudu/127.9.143.252@KRBTEST.COM
08:08:10.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.723857 9892 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f: Bootstrap starting.
08:08:10.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.728444 9974 raft_consensus.cc:359] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } }
08:08:10.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.729326 9974 raft_consensus.cc:385] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:08:10.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.729617 9974 raft_consensus.cc:740] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1d36c7228de444508830a92c33ddaa4a, State: Initialized, Role: FOLLOWER
08:08:10.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.730533 9974 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } }
08:08:10.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.731405 9892 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f: Neither blocks nor log segments found. Creating new log.
08:08:10.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.733842 9892 log.cc:826] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f: Log is configured to *not* fsync() on all Append() calls
08:08:10.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.734498 10093 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } } }
08:08:10.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.735653 10093 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a [sys.catalog]: This master's current role is: FOLLOWER
08:08:10.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.737072 9974 sys_catalog.cc:565] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a [sys.catalog]: configured and running, proceeding with master startup.
08:08:10.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.738044 9949 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ca61a7dcebe145b4a80009b389f47393" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1d36c7228de444508830a92c33ddaa4a" is_pre_election: true
08:08:10.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.738961 9949 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ca61a7dcebe145b4a80009b389f47393 in term 0.
08:08:10.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.740567 10012 leader_election.cc:304] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1d36c7228de444508830a92c33ddaa4a, ca61a7dcebe145b4a80009b389f47393; no voters:
08:08:10.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.741636 10075 raft_consensus.cc:2804] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 0 FOLLOWER]: Leader pre-election won for term 1
08:08:10.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.744158 10075 raft_consensus.cc:493] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:08:10.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.744524 10075 raft_consensus.cc:3060] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 0 FOLLOWER]: Advancing to term 1
08:08:10.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.752056 9892 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f: No bootstrap required, opened a new log
08:08:10.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.754650 10075 raft_consensus.cc:515] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } }
08:08:10.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.760200 9949 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ca61a7dcebe145b4a80009b389f47393" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1d36c7228de444508830a92c33ddaa4a"
08:08:10.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.760857 9949 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a [term 0 FOLLOWER]: Advancing to term 1
08:08:10.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.769850 9949 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ca61a7dcebe145b4a80009b389f47393 in term 1.
08:08:10.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.770653 10075 leader_election.cc:290] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [CANDIDATE]: Term 1 election: Requested vote from peers 513a338fca7944dbbf4433e106c2a03f (127.9.143.254:42279), 1d36c7228de444508830a92c33ddaa4a (127.9.143.253:32923)
08:08:10.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.772408 10012 leader_election.cc:304] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1d36c7228de444508830a92c33ddaa4a, ca61a7dcebe145b4a80009b389f47393; no voters:
08:08:10.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.778990 10075 raft_consensus.cc:2804] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 1 FOLLOWER]: Leader election won for term 1
08:08:10.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.780942 10075 raft_consensus.cc:697] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 1 LEADER]: Becoming Leader. State: Replica: ca61a7dcebe145b4a80009b389f47393, State: Running, Role: LEADER
08:08:10.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.783151 10075 consensus_queue.cc:237] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } }
08:08:10.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.790539 9892 raft_consensus.cc:359] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } }
08:08:10.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.791905 9892 raft_consensus.cc:385] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:08:10.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.794334 9892 raft_consensus.cc:740] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 513a338fca7944dbbf4433e106c2a03f, State: Initialized, Role: FOLLOWER
08:08:10.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.795593 9892 consensus_queue.cc:260] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } }
08:08:10.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.805557 10096 sys_catalog.cc:455] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [sys.catalog]: SysCatalogTable state changed. Reason: New leader ca61a7dcebe145b4a80009b389f47393. Latest consensus state: current_term: 1 leader_uuid: "ca61a7dcebe145b4a80009b389f47393" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } } }
08:08:10.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.806679 10096 sys_catalog.cc:458] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [sys.catalog]: This master's current role is: LEADER
08:08:10.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.810416 9866 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ca61a7dcebe145b4a80009b389f47393" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "513a338fca7944dbbf4433e106c2a03f"
08:08:10.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.810541 9867 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ca61a7dcebe145b4a80009b389f47393" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "513a338fca7944dbbf4433e106c2a03f" is_pre_election: true
08:08:10.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.812114 9892 sys_catalog.cc:565] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [sys.catalog]: configured and running, proceeding with master startup.
08:08:10.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.812072 9867 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ca61a7dcebe145b4a80009b389f47393 in term 0.
08:08:10.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.819634 9866 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [term 0 FOLLOWER]: Advancing to term 1
08:08:10.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.812134 10098 sys_catalog.cc:455] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } } }
08:08:10.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.821935 10098 sys_catalog.cc:458] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [sys.catalog]: This master's current role is: FOLLOWER
08:08:10.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.843992 9866 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ca61a7dcebe145b4a80009b389f47393 in term 1.
08:08:10.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.853652 10100 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
08:08:10.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.862356 10100 catalog_manager.cc:1494] Initializing Kudu cluster ID...
08:08:10.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.889444 9949 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a [term 1 FOLLOWER]: Refusing update from remote peer ca61a7dcebe145b4a80009b389f47393: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:08:10.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.891484 10096 consensus_queue.cc:1048] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:08:10.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.901333 9866 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [term 1 FOLLOWER]: Refusing update from remote peer ca61a7dcebe145b4a80009b389f47393: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:08:10.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.903676 10096 consensus_queue.cc:1048] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [LEADER]: Connected to new peer: Peer: permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:08:10.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:10.906800 9846 tablet.cc:2404] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:08:10.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:10.919246 9928 tablet.cc:2404] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:08:10.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.946835 10093 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a [sys.catalog]: SysCatalogTable state changed. Reason: New leader ca61a7dcebe145b4a80009b389f47393. Latest consensus state: current_term: 1 leader_uuid: "ca61a7dcebe145b4a80009b389f47393" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } } }
08:08:10.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.948526 10093 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a [sys.catalog]: This master's current role is: FOLLOWER
08:08:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.951737 10098 sys_catalog.cc:455] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [sys.catalog]: SysCatalogTable state changed. Reason: New leader ca61a7dcebe145b4a80009b389f47393. Latest consensus state: current_term: 1 leader_uuid: "ca61a7dcebe145b4a80009b389f47393" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } } }
08:08:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.952453 10098 sys_catalog.cc:458] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [sys.catalog]: This master's current role is: FOLLOWER
08:08:10.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.957087 10110 mvcc.cc:204] Tried to move back new op lower bound from 7281851149864402944 to 7281851149470732288. Current Snapshot: MvccSnapshot[applied={T|T < 7281851149864402944}]
08:08:10.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.963600 10100 catalog_manager.cc:1357] Generated new cluster ID: 3f6f795373ec4e998985e56bad4e1d1d
08:08:10.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.963948 10100 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
08:08:10.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.965034 10117 mvcc.cc:204] Tried to move back new op lower bound from 7281851149864402944 to 7281851149470732288. Current Snapshot: MvccSnapshot[applied={T|T < 7281851149864402944}]
08:08:10.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.973941 10096 sys_catalog.cc:455] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "ca61a7dcebe145b4a80009b389f47393" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } } }
08:08:10.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.974684 10096 sys_catalog.cc:458] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [sys.catalog]: This master's current role is: LEADER
08:08:10.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.975167 10096 sys_catalog.cc:455] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "ca61a7dcebe145b4a80009b389f47393" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } } }
08:08:10.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.975761 10096 sys_catalog.cc:458] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [sys.catalog]: This master's current role is: LEADER
08:08:10.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.981019 10093 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "ca61a7dcebe145b4a80009b389f47393" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } } }
08:08:10.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.982919 10093 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a [sys.catalog]: This master's current role is: FOLLOWER
08:08:10.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:10.981058 10129 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
08:08:10.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:10.992743 10124 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:08:10.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:10.993083 10124 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:08:11.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:11.002336 10098 sys_catalog.cc:455] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "ca61a7dcebe145b4a80009b389f47393" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } } }
08:08:11.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:11.003054 10098 sys_catalog.cc:458] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [sys.catalog]: This master's current role is: FOLLOWER
08:08:11.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:11.079078 10100 catalog_manager.cc:1380] Generated new certificate authority record
08:08:11.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:11.081209 10100 catalog_manager.cc:1514] Loading token signing keys...
08:08:11.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:11.107364 10100 catalog_manager.cc:6044] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393: Generated new TSK 0
08:08:11.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:11.108501 10100 catalog_manager.cc:1524] Initializing in-progress tserver states...
08:08:11.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:11.998939 10124 catalog_manager.cc:1269] Loaded cluster ID: 3f6f795373ec4e998985e56bad4e1d1d
08:08:11.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:11.999498 10124 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f: loading cluster ID for follower catalog manager: success
08:08:12.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:12.004733 10129 catalog_manager.cc:1269] Loaded cluster ID: 3f6f795373ec4e998985e56bad4e1d1d
08:08:12.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:12.005043 10129 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a: loading cluster ID for follower catalog manager: success
08:08:12.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:12.014393 10124 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f: acquiring CA information for follower catalog manager: success
08:08:12.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:12.020195 10129 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a: acquiring CA information for follower catalog manager: success
08:08:12.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:12.026489 10129 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:08:12.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:12.031921 10124 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:08:13.367 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.9.143.252:37215
08:08:13.367 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
08:08:13.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:13.369894 9791 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu with pid 9984
08:08:13.395 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:08:13.395 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.9.143.254:42279
08:08:13.395 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
08:08:13.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:13.396023 9791 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu with pid 9821
08:08:13.433 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:08:13.433 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.9.143.253:32923
08:08:13.433 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
08:08:13.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:13.433709 9791 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu with pid 9901
08:08:13.457 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:08:13.457 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.9.143.252:37215
08:08:13.457 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
08:08:13.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:13.458940 9791 external_mini_cluster.cc:1366] Running /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu
08:08:13.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu
08:08:13.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/wal
08:08:13.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/data
08:08:13.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:08:13.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:08:13.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:08:13.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:13.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:13.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:13.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/logs
08:08:13.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/data/info.pb
08:08:13.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:13.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:08:13.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:08:13.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:08:13.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:08:13.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:13.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:13.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:13.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:08:13.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:08:13.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:08:13.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:08:13.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.143.252:37215
08:08:13.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.143.252
08:08:13.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45863
08:08:13.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kudu.keytab
08:08:13.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.143.252
08:08:13.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:08:13.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:08:13.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:08:13.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:08:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.143.212:44461
08:08:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:08:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.143.254:42279,127.9.143.253:32923,127.9.143.252:37215
08:08:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:08:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36329/jwks.json
08:08:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:08:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:08:13.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:13.780103 10138 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:08:13.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:13.780714 10138 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:08:13.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:13.781344 10138 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:08:13.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:13.815582 10138 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
08:08:13.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:13.815850 10138 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:08:13.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:13.816136 10138 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:08:13.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:13.816344 10138 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:08:13.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:13.816581 10138 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:08:13.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:13.816780 10138 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:08:13.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:13.816928 10138 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:36329/jwks.json
08:08:13.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:13.857028 10138 master_runner.cc:386] Master server non-default flags:
08:08:13.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:13.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.143.212:44461
08:08:13.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:13.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:13.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/data
08:08:13.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/wal
08:08:13.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:08:13.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:08:13.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.143.254:42279,127.9.143.253:32923,127.9.143.252:37215
08:08:13.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kudu.keytab
08:08:13.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.143.252
08:08:13.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:13.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:13.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:08:13.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.143.252:37215
08:08:13.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:08:13.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:08:13.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:08:13.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36329/jwks.json
08:08:13.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:08:13.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:08:13.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:08:13.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:13.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:13.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/data/info.pb
08:08:13.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.143.252
08:08:13.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45863
08:08:13.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:08:13.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:13.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:08:13.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:08:13.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:13.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/logs
08:08:13.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:13.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:08:13.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:08:13.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:08:13.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:08:13.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
08:08:13.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:08:13.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 07:43:21 UTC on e7f111948823
08:08:13.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11723
08:08:13.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:08:13.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:13.858433 10138 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:08:13.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:13.859504 10138 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:08:13.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:13.871165 10144 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:15.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:13.872339 10145 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:15.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:13.874887 10147 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:15.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:15.080413 10146 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1204 milliseconds
08:08:15.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:15.080519 10138 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:08:15.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:15.081634 10138 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:08:15.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:15.084506 10138 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:08:15.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:15.085938 10138 hybrid_clock.cc:648] HybridClock initialized: now 1777795695085888 us; error 60 us; skew 500 ppm
08:08:15.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:15.086829 10138 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:08:15.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:08:15 dist-test-slave-tr3r krb5kdc[9803](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777795695, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.143.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:08:15.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:15.092820 10138 init.cc:377] Logged in from keytab as kudu/127.9.143.252@KRBTEST.COM (short username kudu)
08:08:15.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:15.097826 10138 webserver.cc:492] Webserver started at http://127.9.143.252:45863/ using document root <none> and password file <none>
08:08:15.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:15.098694 10138 fs_manager.cc:362] Metadata directory not provided
08:08:15.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:15.098887 10138 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:08:15.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:15.106854 10138 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.007s sys 0.000s
08:08:15.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:15.111675 10156 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:08:15.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:15.112972 10138 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.001s
08:08:15.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:15.113312 10138 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/data,/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/wal
08:08:15.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ca61a7dcebe145b4a80009b389f47393"
08:08:15.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 08:08:10 on dist-test-slave-tr3r"
08:08:15.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:15.115375 10138 fs_report.cc:389] FS layout report
08:08:15.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:08:15.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/wal
08:08:15.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/wal
08:08:15.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/data/data
08:08:15.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:08:15.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:08:15.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:08:15.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:08:15.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:08:15.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:08:15.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:08:15.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:08:15.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:08:15.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:08:15.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:08:15.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:15.162806 10138 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:08:15.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:15.171263 10138 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:08:15.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:15.171684 10138 kserver.cc:163] Server-wide thread pool size limit: 3276
08:08:15.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:15.244784 10138 rpc_server.cc:307] RPC server started. Bound to: 127.9.143.252:37215
08:08:15.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:15.244850 10207 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.143.252:37215 every 8 connection(s)
08:08:15.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:15.247611 10138 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/data/info.pb
08:08:15.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:15.251451 9791 external_mini_cluster.cc:1428] Started /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu as pid 10138
08:08:15.252 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:08:15.252 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.9.143.254:42279
08:08:15.252 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
08:08:15.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:15.254283 9791 external_mini_cluster.cc:1366] Running /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu
08:08:15.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu
08:08:15.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/wal
08:08:15.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/data
08:08:15.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:08:15.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:08:15.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:08:15.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:15.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:15.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:15.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/logs
08:08:15.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/data/info.pb
08:08:15.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:15.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:08:15.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:08:15.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:08:15.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:08:15.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:15.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:15.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:15.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:08:15.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:08:15.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:08:15.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:08:15.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.143.254:42279
08:08:15.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.143.254
08:08:15.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42699
08:08:15.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kudu.keytab
08:08:15.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.143.254
08:08:15.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:08:15.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:08:15.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:08:15.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:08:15.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.143.212:44461
08:08:15.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:15.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:15.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:15.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:08:15.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.143.254:42279,127.9.143.253:32923,127.9.143.252:37215
08:08:15.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:08:15.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36329/jwks.json
08:08:15.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:08:15.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:08:15.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:15.260984 10208 sys_catalog.cc:263] Verifying existing consensus state
08:08:15.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:15.266363 10208 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393: Bootstrap starting.
08:08:15.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:15.304170 10208 log.cc:826] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393: Log is configured to *not* fsync() on all Append() calls
08:08:15.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:15.320539 10208 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:08:15.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:15.321336 10208 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393: Bootstrap complete.
08:08:15.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:15.343848 10208 raft_consensus.cc:359] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } }
08:08:15.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:15.346103 10208 raft_consensus.cc:740] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: ca61a7dcebe145b4a80009b389f47393, State: Initialized, Role: FOLLOWER
08:08:15.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:15.346968 10208 consensus_queue.cc:260] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } }
08:08:15.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:15.349808 10212 sys_catalog.cc:455] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } } }
08:08:15.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:15.350637 10212 sys_catalog.cc:458] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [sys.catalog]: This master's current role is: FOLLOWER
08:08:15.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:15.351462 10208 sys_catalog.cc:565] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [sys.catalog]: configured and running, proceeding with master startup.
08:08:15.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:15.379570 10223 catalog_manager.cc:1269] Loaded cluster ID: 3f6f795373ec4e998985e56bad4e1d1d
08:08:15.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:15.379853 10223 catalog_manager.cc:1562] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393: loading cluster ID for follower catalog manager: success
08:08:15.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:15.387218 10223 catalog_manager.cc:1584] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393: acquiring CA information for follower catalog manager: success
08:08:15.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:15.392554 10223 catalog_manager.cc:1612] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:08:15.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:15.376317 10138 master_runner.cc:186] Error getting master registration for 127.9.143.254:42279: Network error: Client connection negotiation failed: client connection to 127.9.143.254:42279: connect: Connection refused (error 111),
08:08:15.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:15.403575 10138 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:08:15.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:15.575838 10210 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:08:15.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:15.576367 10210 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:08:15.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:15.576978 10210 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:08:15.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:15.633314 10210 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
08:08:15.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:15.633652 10210 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:08:15.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:15.634164 10210 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:08:15.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:15.634521 10210 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:08:15.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:15.634850 10210 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:08:15.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:15.635174 10210 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:08:15.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:15.635429 10210 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:36329/jwks.json
08:08:15.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:15.699563 10210 master_runner.cc:386] Master server non-default flags:
08:08:15.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:15.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.143.212:44461
08:08:15.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:15.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:15.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/data
08:08:15.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/wal
08:08:15.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:08:15.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:08:15.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.143.254:42279,127.9.143.253:32923,127.9.143.252:37215
08:08:15.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kudu.keytab
08:08:15.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.143.254
08:08:15.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:15.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:15.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:08:15.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.143.254:42279
08:08:15.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:08:15.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:08:15.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:08:15.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36329/jwks.json
08:08:15.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:08:15.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:08:15.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:08:15.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:15.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:15.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/data/info.pb
08:08:15.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.143.254
08:08:15.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42699
08:08:15.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:08:15.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:15.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:08:15.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:08:15.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:15.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/logs
08:08:15.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:15.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:08:15.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:08:15.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:08:15.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:08:15.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
08:08:15.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:08:15.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 07:43:21 UTC on e7f111948823
08:08:15.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11723
08:08:15.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:08:15.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:15.700814 10210 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:08:15.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:15.701779 10210 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:08:15.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:15.713567 10231 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:16.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:16.651325 10237 raft_consensus.cc:493] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:08:16.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:16.651978 10237 raft_consensus.cc:515] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } }
08:08:16.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:16.687827 10158 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.143.253:32923: connect: Connection refused (error 111)
08:08:16.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:16.692312 10237 leader_election.cc:290] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 513a338fca7944dbbf4433e106c2a03f (127.9.143.254:42279), 1d36c7228de444508830a92c33ddaa4a (127.9.143.253:32923)
08:08:16.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:16.703764 10158 leader_election.cc:336] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 1d36c7228de444508830a92c33ddaa4a (127.9.143.253:32923): Network error: Client connection negotiation failed: client connection to 127.9.143.253:32923: connect: Connection refused (error 111)
08:08:16.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:16.704694 10159 leader_election.cc:336] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 513a338fca7944dbbf4433e106c2a03f (127.9.143.254:42279): Network error: Client connection negotiation failed: client connection to 127.9.143.254:42279: connect: Connection refused (error 111)
08:08:16.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:16.705134 10159 leader_election.cc:304] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: ca61a7dcebe145b4a80009b389f47393; no voters: 1d36c7228de444508830a92c33ddaa4a, 513a338fca7944dbbf4433e106c2a03f
08:08:16.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:16.706174 10237 raft_consensus.cc:2749] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
08:08:17.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:17.115450 10230 debug-util.cc:398] Leaking SignalData structure 0x7b0800028d00 after lost signal to thread 10210
08:08:17.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:17.342327 10230 kernel_stack_watchdog.cc:198] Thread 10210 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 398ms:
08:08:17.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:08:17.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:08:17.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:08:17.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:08:17.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:15.715129 10232 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:17.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:17.343850 10233 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1627 milliseconds
08:08:17.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:17.343811 10210 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.628s user 0.510s sys 0.904s
08:08:17.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:17.344481 10210 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.629s user 0.510s sys 0.904s
08:08:17.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:17.344996 10234 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:17.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:17.344949 10210 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:08:17.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:17.351379 10210 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:08:17.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:17.354034 10210 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:08:17.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:17.357640 10210 hybrid_clock.cc:648] HybridClock initialized: now 1777795697357569 us; error 67 us; skew 500 ppm
08:08:17.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:17.358630 10210 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:08:17.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:08:17 dist-test-slave-tr3r krb5kdc[9803](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777795697, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.143.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:08:17.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:17.364701 10210 init.cc:377] Logged in from keytab as kudu/127.9.143.254@KRBTEST.COM (short username kudu)
08:08:17.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:17.370215 10210 webserver.cc:492] Webserver started at http://127.9.143.254:42699/ using document root <none> and password file <none>
08:08:17.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:17.371184 10210 fs_manager.cc:362] Metadata directory not provided
08:08:17.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:17.371438 10210 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:08:17.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:17.380141 10210 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.001s sys 0.004s
08:08:17.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:17.384891 10246 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:08:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:17.386250 10210 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.000s
08:08:17.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:17.386643 10210 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/data,/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/wal
08:08:17.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "513a338fca7944dbbf4433e106c2a03f"
08:08:17.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 08:08:05 on dist-test-slave-tr3r"
08:08:17.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:17.388612 10210 fs_report.cc:389] FS layout report
08:08:17.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:08:17.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/wal
08:08:17.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/wal
08:08:17.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/data/data
08:08:17.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:08:17.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:08:17.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:08:17.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:08:17.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:08:17.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:08:17.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:08:17.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:08:17.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:08:17.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:08:17.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:08:17.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:17.440840 10210 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:08:17.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:17.447126 10210 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:08:17.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:17.447628 10210 kserver.cc:163] Server-wide thread pool size limit: 3276
08:08:17.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:17.528165 10210 rpc_server.cc:307] RPC server started. Bound to: 127.9.143.254:42279
08:08:17.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:17.528270 10297 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.143.254:42279 every 8 connection(s)
08:08:17.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:17.532006 10210 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/data/info.pb
08:08:17.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:17.537020 9791 external_mini_cluster.cc:1428] Started /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu as pid 10210
08:08:17.537 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:08:17.537 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.9.143.253:32923
08:08:17.538 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
08:08:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:17.539692 9791 external_mini_cluster.cc:1366] Running /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu
08:08:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu
08:08:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/wal
08:08:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/data
08:08:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:08:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:08:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:08:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/logs
08:08:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/data/info.pb
08:08:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:08:17.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:08:17.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:08:17.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:08:17.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:17.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:17.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:17.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:08:17.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:08:17.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:08:17.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:08:17.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.143.253:32923
08:08:17.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.143.253
08:08:17.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37721
08:08:17.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kudu.keytab
08:08:17.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.143.253
08:08:17.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:08:17.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:08:17.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:08:17.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:08:17.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.143.212:44461
08:08:17.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:17.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:17.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:17.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:08:17.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.143.254:42279,127.9.143.253:32923,127.9.143.252:37215
08:08:17.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:08:17.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36329/jwks.json
08:08:17.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:08:17.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:08:17.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:17.559235 10298 sys_catalog.cc:263] Verifying existing consensus state
08:08:17.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:17.568208 10298 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f: Bootstrap starting.
08:08:17.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:17.637669 10298 log.cc:826] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f: Log is configured to *not* fsync() on all Append() calls
08:08:17.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:17.664965 10298 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:08:17.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:17.666070 10298 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f: Bootstrap complete.
08:08:17.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:17.699820 10298 raft_consensus.cc:359] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } }
08:08:17.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:17.703222 10298 raft_consensus.cc:740] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 513a338fca7944dbbf4433e106c2a03f, State: Initialized, Role: FOLLOWER
08:08:17.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:17.704249 10298 consensus_queue.cc:260] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } }
08:08:17.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:17.707024 10303 sys_catalog.cc:455] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } } }
08:08:17.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:17.708034 10303 sys_catalog.cc:458] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [sys.catalog]: This master's current role is: FOLLOWER
08:08:17.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:17.708997 10298 sys_catalog.cc:565] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [sys.catalog]: configured and running, proceeding with master startup.
08:08:17.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:17.768628 10314 catalog_manager.cc:1269] Loaded cluster ID: 3f6f795373ec4e998985e56bad4e1d1d
08:08:17.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:17.768949 10314 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f: loading cluster ID for follower catalog manager: success
08:08:17.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:17.787626 10314 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f: acquiring CA information for follower catalog manager: success
08:08:17.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:08:17 dist-test-slave-tr3r krb5kdc[9803](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777795697, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.143.254@KRBTEST.COM for kudu/127.9.143.254@KRBTEST.COM
08:08:17.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:17.799362 10314 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:08:17.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:17.829720 10210 master_runner.cc:186] Error getting master registration for 127.9.143.253:32923: Network error: Client connection negotiation failed: client connection to 127.9.143.253:32923: connect: Connection refused (error 111),
08:08:17.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:17.853610 10210 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:08:18.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:18.014847 10301 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:08:18.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:18.015880 10301 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:08:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:18.016945 10301 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:08:18.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:18.080075 10301 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
08:08:18.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:18.080420 10301 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:08:18.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:18.080906 10301 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:08:18.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:18.081254 10301 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:08:18.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:18.081601 10301 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:08:18.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:18.081965 10301 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:08:18.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:18.082223 10301 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:36329/jwks.json
08:08:18.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:18.135150 10301 master_runner.cc:386] Master server non-default flags:
08:08:18.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:18.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.143.212:44461
08:08:18.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:18.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:18.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/data
08:08:18.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/wal
08:08:18.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:08:18.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:08:18.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.143.254:42279,127.9.143.253:32923,127.9.143.252:37215
08:08:18.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kudu.keytab
08:08:18.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.143.253
08:08:18.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:18.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:18.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:08:18.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.143.253:32923
08:08:18.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:08:18.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:08:18.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:08:18.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36329/jwks.json
08:08:18.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:08:18.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:08:18.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:08:18.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:18.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:18.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/data/info.pb
08:08:18.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.143.253
08:08:18.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37721
08:08:18.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:08:18.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:18.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:08:18.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:08:18.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:18.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/logs
08:08:18.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:18.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:08:18.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:08:18.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:08:18.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:08:18.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
08:08:18.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:08:18.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 07:43:21 UTC on e7f111948823
08:08:18.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11723
08:08:18.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:08:18.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:18.136590 10301 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:08:18.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:18.138046 10301 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:08:18.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:18.161707 10324 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:18.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:18.472838 10329 raft_consensus.cc:493] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:08:18.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:18.473870 10329 raft_consensus.cc:515] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } }
08:08:18.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:18.490021 10329 leader_election.cc:290] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 513a338fca7944dbbf4433e106c2a03f (127.9.143.254:42279), 1d36c7228de444508830a92c33ddaa4a (127.9.143.253:32923)
08:08:18.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:18.518137 10273 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ca61a7dcebe145b4a80009b389f47393" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "513a338fca7944dbbf4433e106c2a03f" is_pre_election: true
08:08:18.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:18.519299 10273 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ca61a7dcebe145b4a80009b389f47393 in term 1.
08:08:18.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:18.520843 10159 leader_election.cc:304] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 513a338fca7944dbbf4433e106c2a03f, ca61a7dcebe145b4a80009b389f47393; no voters:
08:08:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:18.521667 10329 raft_consensus.cc:2804] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 1 FOLLOWER]: Leader pre-election won for term 2
08:08:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:18.522552 10329 raft_consensus.cc:493] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:08:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:18.522959 10329 raft_consensus.cc:3060] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 1 FOLLOWER]: Advancing to term 2
08:08:18.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:18.538776 10158 leader_election.cc:336] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 1d36c7228de444508830a92c33ddaa4a (127.9.143.253:32923): Network error: Client connection negotiation failed: client connection to 127.9.143.253:32923: connect: Connection refused (error 111)
08:08:18.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:18.554725 10329 raft_consensus.cc:515] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } }
08:08:18.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:18.563732 10273 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ca61a7dcebe145b4a80009b389f47393" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "513a338fca7944dbbf4433e106c2a03f"
08:08:18.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:18.564302 10273 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [term 1 FOLLOWER]: Advancing to term 2
08:08:18.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:18.570439 10158 leader_election.cc:336] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 1d36c7228de444508830a92c33ddaa4a (127.9.143.253:32923): Network error: Client connection negotiation failed: client connection to 127.9.143.253:32923: connect: Connection refused (error 111)
08:08:18.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:18.572192 10329 leader_election.cc:290] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [CANDIDATE]: Term 2 election: Requested vote from peers 513a338fca7944dbbf4433e106c2a03f (127.9.143.254:42279), 1d36c7228de444508830a92c33ddaa4a (127.9.143.253:32923)
08:08:18.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:18.573328 10273 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ca61a7dcebe145b4a80009b389f47393 in term 2.
08:08:18.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:18.579092 10159 leader_election.cc:304] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 513a338fca7944dbbf4433e106c2a03f, ca61a7dcebe145b4a80009b389f47393; no voters: 1d36c7228de444508830a92c33ddaa4a
08:08:18.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:18.580968 10329 raft_consensus.cc:2804] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 2 FOLLOWER]: Leader election won for term 2
08:08:18.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:18.592379 10329 raft_consensus.cc:697] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 2 LEADER]: Becoming Leader. State: Replica: ca61a7dcebe145b4a80009b389f47393, State: Running, Role: LEADER
08:08:18.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:18.593925 10329 consensus_queue.cc:237] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } }
08:08:18.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:18.641485 10334 sys_catalog.cc:455] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [sys.catalog]: SysCatalogTable state changed. Reason: New leader ca61a7dcebe145b4a80009b389f47393. Latest consensus state: current_term: 2 leader_uuid: "ca61a7dcebe145b4a80009b389f47393" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } } }
08:08:18.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:18.642460 10334 sys_catalog.cc:458] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [sys.catalog]: This master's current role is: LEADER
08:08:18.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:18.647457 10336 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
08:08:18.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:18.656144 10336 catalog_manager.cc:1494] Initializing Kudu cluster ID...
08:08:18.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:18.660518 10336 catalog_manager.cc:1269] Loaded cluster ID: 3f6f795373ec4e998985e56bad4e1d1d
08:08:18.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:18.660864 10336 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
08:08:18.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:18.666087 10336 catalog_manager.cc:1514] Loading token signing keys...
08:08:18.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:18.670485 10336 catalog_manager.cc:6055] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393: Loaded TSK: 0
08:08:18.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:18.672089 10336 catalog_manager.cc:1524] Initializing in-progress tserver states...
08:08:19.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:19.070083 10158 consensus_peers.cc:597] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 -> Peer 1d36c7228de444508830a92c33ddaa4a (127.9.143.253:32923): Couldn't send request to peer 1d36c7228de444508830a92c33ddaa4a. Status: Network error: Client connection negotiation failed: client connection to 127.9.143.253:32923: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:08:19.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:19.101207 10273 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [term 2 FOLLOWER]: Refusing update from remote peer ca61a7dcebe145b4a80009b389f47393: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
08:08:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:19.103555 10334 consensus_queue.cc:1048] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [LEADER]: Connected to new peer: Peer: permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
08:08:19.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:19.132352 10337 sys_catalog.cc:455] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [sys.catalog]: SysCatalogTable state changed. Reason: New leader ca61a7dcebe145b4a80009b389f47393. Latest consensus state: current_term: 2 leader_uuid: "ca61a7dcebe145b4a80009b389f47393" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } } }
08:08:19.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:19.133060 10337 sys_catalog.cc:458] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [sys.catalog]: This master's current role is: FOLLOWER
08:08:19.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:19.139143 10334 sys_catalog.cc:455] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "ca61a7dcebe145b4a80009b389f47393" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } } }
08:08:19.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:19.141496 10334 sys_catalog.cc:458] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [sys.catalog]: This master's current role is: LEADER
08:08:19.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:19.143244 10337 sys_catalog.cc:455] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "ca61a7dcebe145b4a80009b389f47393" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } } }
08:08:19.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:19.143889 10337 sys_catalog.cc:458] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [sys.catalog]: This master's current role is: FOLLOWER
08:08:19.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:18.162335 10325 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:19.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:19.556277 10301 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.397s user 0.519s sys 0.724s
08:08:19.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:19.567639 10301 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.408s user 0.520s sys 0.725s
08:08:19.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:19.578439 10301 server_base.cc:1061] running on GCE node
08:08:19.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:19.580626 10342 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:19.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:19.586668 10301 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:08:19.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:19.646345 10301 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:08:19.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:19.650085 10301 hybrid_clock.cc:648] HybridClock initialized: now 1777795699650023 us; error 63 us; skew 500 ppm
08:08:19.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:19.651317 10301 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:08:19.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:08:19 dist-test-slave-tr3r krb5kdc[9803](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777795699, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.143.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:08:19.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:19.661224 10301 init.cc:377] Logged in from keytab as kudu/127.9.143.253@KRBTEST.COM (short username kudu)
08:08:19.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:19.683211 10301 webserver.cc:492] Webserver started at http://127.9.143.253:37721/ using document root <none> and password file <none>
08:08:19.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:19.684254 10301 fs_manager.cc:362] Metadata directory not provided
08:08:19.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:19.684497 10301 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:08:19.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:19.693079 10301 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.005s sys 0.001s
08:08:19.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:19.698453 10348 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:08:19.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:19.699712 10301 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s
08:08:19.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:19.700124 10301 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/data,/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/wal
08:08:19.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1d36c7228de444508830a92c33ddaa4a"
08:08:19.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 08:08:08 on dist-test-slave-tr3r"
08:08:19.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:19.702212 10301 fs_report.cc:389] FS layout report
08:08:19.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:08:19.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/wal
08:08:19.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/wal
08:08:19.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/data/data
08:08:19.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:08:19.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:08:19.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:08:19.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:08:19.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:08:19.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:08:19.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:08:19.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:08:19.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:08:19.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:08:19.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:08:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:19.760164 10301 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:08:19.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:19.766067 10301 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:08:19.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:19.766515 10301 kserver.cc:163] Server-wide thread pool size limit: 3276
08:08:19.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:19.855841 10301 rpc_server.cc:307] RPC server started. Bound to: 127.9.143.253:32923
08:08:19.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:19.855882 10399 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.143.253:32923 every 8 connection(s)
08:08:19.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:19.859890 10301 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/data/info.pb
08:08:19.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:19.867883 9791 external_mini_cluster.cc:1428] Started /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu as pid 10301
08:08:19.868 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:08:19.878 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.9.143.252:37215(127.9.143.252:37215)] connecting to peer
08:08:19.879 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.9.143.254:42279(127.9.143.254:42279)] connecting to peer
08:08:19.883 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.9.143.252:37215(127.9.143.252:37215)] Successfully connected to peer
08:08:19.886 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.9.143.253:32923(127.9.143.253:32923)] connecting to peer
08:08:19.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:19.895573 10400 sys_catalog.cc:263] Verifying existing consensus state
08:08:19.902 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.9.143.254:42279(127.9.143.254:42279)] Successfully connected to peer
08:08:19.910 [DEBUG - kudu-nio-0] (SecurityContext.java:222) Refreshing Kerberos credentials...
08:08:19.910 [DEBUG - kudu-nio-0] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/krb5cc
08:08:19.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:19.914750 10400 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a: Bootstrap starting.
08:08:19.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:08:19 dist-test-slave-tr3r krb5kdc[9803](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777795695, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.143.252@KRBTEST.COM for kudu/127.9.143.253@KRBTEST.COM
08:08:19.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:08:19 dist-test-slave-tr3r krb5kdc[9803](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1777795690, etypes {rep=17 tkt=17 ses=17}, test-user@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:08:19.925 [DEBUG - kudu-nio-0] (SecurityUtil.java:119) Logged in as subject: test-user@KRBTEST.COM
08:08:19.925 [ERROR - kudu-nio-0] (SecurityContext.java:246) Attempted to refresh Kerberos credentials from ticket cache but found that the new Kerberos principal test-user@KRBTEST.COM did not match the original principal test-admin@KRBTEST.COM. Ignoring.
08:08:19.926 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.9.143.254
08:08:19.933 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.9.143.252
08:08:19.939 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.9.143.253:32923(127.9.143.253:32923)] Successfully connected to peer
08:08:19.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:08:19 dist-test-slave-tr3r krb5kdc[9803](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1777795690, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.9.143.254@KRBTEST.COM
08:08:19.977 [DEBUG - kudu-nio-2] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.9.143.253
08:08:19.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:19.981163 10400 log.cc:826] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a: Log is configured to *not* fsync() on all Append() calls
08:08:19.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:08:19 dist-test-slave-tr3r krb5kdc[9803](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1777795690, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.9.143.252@KRBTEST.COM
08:08:20.016 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xd10784cb, L:/127.0.0.1:38422 - R:/127.9.143.252:37215] using SASL/GSSAPI
08:08:20.020 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x8e9b57e3, L:/127.0.0.1:58294 - R:/127.9.143.254:42279] using SASL/GSSAPI
08:08:20.029 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.9.143.253:32923(127.9.143.253:32923)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
08:08:20.030 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.9.143.253:32923: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
08:08:20.031 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.9.143.253:32923(127.9.143.253:32923)] handling channelInactive
08:08:20.031 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
08:08:20.031 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.143.252:37215, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.143.254:42279, [9ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.143.253:32923, [148ms] Sub RPC ConnectToMaster: received response from server master-127.9.143.254:42279: OK, [153ms] Sub RPC ConnectToMaster: received response from server master-127.9.143.253:32923: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [153ms] Sub RPC ConnectToMaster: received response from server master-127.9.143.252:37215: OK, deferred=null) after lookup
08:08:20.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:20.038656 10410 negotiation.cc:336] Failed RPC negotiation. Trace:
08:08:20.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 08:08:19.936754 (+ 0us) reactor.cc:678] Submitting negotiation task for server connection from 127.0.0.1:42968 (local address 127.9.143.253:32923)
08:08:20.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 08:08:19.953383 (+ 16629us) server_negotiation.cc:207] Beginning negotiation
08:08:20.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 08:08:19.953413 (+ 30us) server_negotiation.cc:400] Waiting for connection header
08:08:20.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 08:08:19.971575 (+ 18162us) server_negotiation.cc:408] Connection header received
08:08:20.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 08:08:19.972656 (+ 1081us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
08:08:20.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 08:08:19.972865 (+ 209us) server_negotiation.cc:462] Received NEGOTIATE request from client
08:08:20.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 08:08:19.974369 (+ 1504us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
08:08:20.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 08:08:19.986355 (+ 11986us) server_negotiation.cc:227] Negotiated authn=SASL
08:08:20.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 08:08:19.997320 (+ 10965us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
08:08:20.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 08:08:20.000448 (+ 3128us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
08:08:20.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 08:08:20.029125 (+ 28677us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
08:08:20.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 08:08:20.029725 (+ 600us) server_negotiation.cc:663] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
08:08:20.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 08:08:20.036521 (+ 6796us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:42968: BlockingRecv error: failed to read from TLS socket (remote: unknown): Cannot send after transport endpoint shutdown (error 108)
08:08:20.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":15936,"thread_start_us":13817,"threads_started":1}
08:08:20.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:20.049461 10400 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:08:20.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:20.050558 10400 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a: Bootstrap complete.
08:08:20.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:20.081179 10400 raft_consensus.cc:359] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } }
08:08:20.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:20.084781 10400 raft_consensus.cc:740] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1d36c7228de444508830a92c33ddaa4a, State: Initialized, Role: FOLLOWER
08:08:20.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:20.085767 10400 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } }
08:08:20.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:20.088660 10413 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } } }
08:08:20.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:20.089464 10413 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a [sys.catalog]: This master's current role is: FOLLOWER
08:08:20.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:20.090350 10400 sys_catalog.cc:565] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a [sys.catalog]: configured and running, proceeding with master startup.
08:08:20.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:20.091056 10375 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a [term 1 FOLLOWER]: Advancing to term 2
08:08:20.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:20.131001 10425 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "ca61a7dcebe145b4a80009b389f47393" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } } }
08:08:20.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:20.132004 10425 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a [sys.catalog]: This master's current role is: FOLLOWER
08:08:20.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:20.135739 10413 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a [sys.catalog]: SysCatalogTable state changed. Reason: New leader ca61a7dcebe145b4a80009b389f47393. Latest consensus state: current_term: 2 leader_uuid: "ca61a7dcebe145b4a80009b389f47393" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } } }
08:08:20.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:20.137921 10413 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a [sys.catalog]: This master's current role is: FOLLOWER
08:08:20.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:20.147689 10427 catalog_manager.cc:1269] Loaded cluster ID: 3f6f795373ec4e998985e56bad4e1d1d
08:08:20.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:20.147987 10427 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a: loading cluster ID for follower catalog manager: success
08:08:20.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:08:20 dist-test-slave-tr3r krb5kdc[9803](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777795699, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.143.253@KRBTEST.COM for kudu/127.9.143.254@KRBTEST.COM
08:08:20.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:20.155364 10427 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a: acquiring CA information for follower catalog manager: success
08:08:20.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:20.160898 10427 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:08:20.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:20.436311 10334 sys_catalog.cc:455] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "ca61a7dcebe145b4a80009b389f47393" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } } }
08:08:20.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:20.437343 10334 sys_catalog.cc:458] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [sys.catalog]: This master's current role is: LEADER
08:08:23.084 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.9.143.252:37215
08:08:23.085 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
08:08:23.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:23.085772 9791 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu with pid 10138
08:08:23.110 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.9.143.252:37215(127.9.143.252:37215)] handling channelInactive
08:08:23.110 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:08:23.111 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.9.143.254:42279
08:08:23.111 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.143.252:37215(127.9.143.252:37215)] cleaning up while in state READY due to: connection closed
08:08:23.111 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
08:08:23.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:23.112214 9791 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu with pid 10210
08:08:23.135 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.9.143.254:42279(127.9.143.254:42279)] handling channelInactive
08:08:23.135 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.143.254:42279(127.9.143.254:42279)] cleaning up while in state READY due to: connection closed
08:08:23.135 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:08:23.135 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.9.143.253:32923
08:08:23.136 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
08:08:23.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:23.136412 9791 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu with pid 10301
08:08:23.159 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:08:23.159 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.9.143.252:37215
08:08:23.160 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
08:08:23.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:23.161877 9791 external_mini_cluster.cc:1366] Running /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu
08:08:23.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu
08:08:23.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/wal
08:08:23.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/data
08:08:23.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:08:23.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:08:23.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:08:23.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:23.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:23.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:23.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/logs
08:08:23.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/data/info.pb
08:08:23.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:23.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:08:23.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:08:23.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:08:23.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:08:23.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:23.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:23.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:23.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:08:23.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:08:23.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:08:23.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:08:23.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.143.252:37215
08:08:23.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.143.252
08:08:23.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45863
08:08:23.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kudu.keytab
08:08:23.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.143.252
08:08:23.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:08:23.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:08:23.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:08:23.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:08:23.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.143.212:44461
08:08:23.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:23.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:23.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:23.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:08:23.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.143.254:42279,127.9.143.253:32923,127.9.143.252:37215
08:08:23.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:08:23.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36329/jwks.json
08:08:23.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:08:23.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:08:23.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:23.473665 10442 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:08:23.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:23.474545 10442 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:08:23.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:23.475598 10442 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:08:23.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:23.511427 10442 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
08:08:23.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:23.511672 10442 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:08:23.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:23.511943 10442 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:08:23.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:23.512167 10442 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:08:23.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:23.512456 10442 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:08:23.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:23.512657 10442 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:08:23.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:23.512836 10442 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:36329/jwks.json
08:08:23.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:23.552142 10442 master_runner.cc:386] Master server non-default flags:
08:08:23.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:23.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.143.212:44461
08:08:23.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:23.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:23.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/data
08:08:23.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/wal
08:08:23.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:08:23.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:08:23.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.143.254:42279,127.9.143.253:32923,127.9.143.252:37215
08:08:23.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kudu.keytab
08:08:23.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.143.252
08:08:23.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:23.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:23.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:08:23.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.143.252:37215
08:08:23.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:08:23.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:08:23.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:08:23.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36329/jwks.json
08:08:23.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:08:23.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:08:23.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:08:23.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:23.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:23.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/data/info.pb
08:08:23.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.143.252
08:08:23.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45863
08:08:23.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:08:23.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:23.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:08:23.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:08:23.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:23.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/logs
08:08:23.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:23.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:08:23.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:08:23.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:08:23.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:08:23.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
08:08:23.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:08:23.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 07:43:21 UTC on e7f111948823
08:08:23.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11723
08:08:23.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:08:23.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:23.553375 10442 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:08:23.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:23.554384 10442 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:08:23.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:23.565795 10448 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:24.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:23.567070 10449 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:24.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:24.751586 10451 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:24.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:24.753619 10450 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1182 milliseconds
08:08:24.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:24.753712 10442 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:08:24.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:24.754969 10442 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:08:24.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:24.757402 10442 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:08:24.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:24.758749 10442 hybrid_clock.cc:648] HybridClock initialized: now 1777795704758697 us; error 56 us; skew 500 ppm
08:08:24.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:24.759594 10442 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:08:24.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:08:24 dist-test-slave-tr3r krb5kdc[9803](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777795704, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.143.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:08:24.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:24.766204 10442 init.cc:377] Logged in from keytab as kudu/127.9.143.252@KRBTEST.COM (short username kudu)
08:08:24.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:24.771360 10442 webserver.cc:492] Webserver started at http://127.9.143.252:45863/ using document root <none> and password file <none>
08:08:24.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:24.772332 10442 fs_manager.cc:362] Metadata directory not provided
08:08:24.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:24.772524 10442 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:08:24.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:24.780417 10442 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.005s sys 0.000s
08:08:24.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:24.784991 10459 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:08:24.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:24.786286 10442 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s
08:08:24.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:24.786630 10442 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/data,/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/wal
08:08:24.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ca61a7dcebe145b4a80009b389f47393"
08:08:24.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 08:08:10 on dist-test-slave-tr3r"
08:08:24.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:24.788807 10442 fs_report.cc:389] FS layout report
08:08:24.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:08:24.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/wal
08:08:24.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/wal
08:08:24.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/data/data
08:08:24.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:08:24.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:08:24.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:08:24.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:08:24.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:08:24.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:08:24.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:08:24.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:08:24.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:08:24.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:08:24.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:08:24.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:24.843868 10442 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:08:24.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:24.853724 10442 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:08:24.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:24.854159 10442 kserver.cc:163] Server-wide thread pool size limit: 3276
08:08:24.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:24.925232 10442 rpc_server.cc:307] RPC server started. Bound to: 127.9.143.252:37215
08:08:24.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:24.925369 10510 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.143.252:37215 every 8 connection(s)
08:08:24.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:24.928081 10442 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/data/info.pb
08:08:24.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:24.928886 9791 external_mini_cluster.cc:1428] Started /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu as pid 10442
08:08:24.929 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:08:24.929 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.9.143.254:42279
08:08:24.929 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
08:08:24.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:24.931232 9791 external_mini_cluster.cc:1366] Running /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu
08:08:24.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu
08:08:24.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/wal
08:08:24.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/data
08:08:24.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:08:24.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:08:24.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:08:24.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:24.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:24.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:24.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/logs
08:08:24.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/data/info.pb
08:08:24.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:24.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:08:24.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:08:24.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:08:24.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:08:24.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:24.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:24.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:24.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:08:24.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:08:24.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:08:24.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:08:24.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.143.254:42279
08:08:24.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.143.254
08:08:24.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42699
08:08:24.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kudu.keytab
08:08:24.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.143.254
08:08:24.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:08:24.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:08:24.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:08:24.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:08:24.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.143.212:44461
08:08:24.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:24.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:24.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:24.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:08:24.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.143.254:42279,127.9.143.253:32923,127.9.143.252:37215
08:08:24.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:08:24.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36329/jwks.json
08:08:24.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:08:24.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:08:24.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:24.941045 10511 sys_catalog.cc:263] Verifying existing consensus state
08:08:24.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:24.947485 10511 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393: Bootstrap starting.
08:08:24.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:24.986368 10511 log.cc:826] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393: Log is configured to *not* fsync() on all Append() calls
08:08:25.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:25.003893 10511 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:08:25.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:25.004698 10511 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393: Bootstrap complete.
08:08:25.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:25.027719 10511 raft_consensus.cc:359] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } }
08:08:25.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:25.030031 10511 raft_consensus.cc:740] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: ca61a7dcebe145b4a80009b389f47393, State: Initialized, Role: FOLLOWER
08:08:25.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:25.030939 10511 consensus_queue.cc:260] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } }
08:08:25.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:25.033798 10516 sys_catalog.cc:455] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } } }
08:08:25.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:25.034869 10516 sys_catalog.cc:458] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [sys.catalog]: This master's current role is: FOLLOWER
08:08:25.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:25.035597 10511 sys_catalog.cc:565] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [sys.catalog]: configured and running, proceeding with master startup.
08:08:25.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:25.067631 10527 catalog_manager.cc:1269] Loaded cluster ID: 3f6f795373ec4e998985e56bad4e1d1d
08:08:25.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:25.067914 10527 catalog_manager.cc:1562] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393: loading cluster ID for follower catalog manager: success
08:08:25.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:25.075985 10527 catalog_manager.cc:1584] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393: acquiring CA information for follower catalog manager: success
08:08:25.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:25.083006 10527 catalog_manager.cc:1612] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:08:25.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:25.061024 10442 master_runner.cc:186] Error getting master registration for 127.9.143.254:42279: Network error: Client connection negotiation failed: client connection to 127.9.143.254:42279: connect: Connection refused (error 111),
08:08:25.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:25.086022 10442 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:08:25.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:25.281972 10513 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:08:25.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:25.282583 10513 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:08:25.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:25.283268 10513 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:08:25.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:25.316736 10513 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
08:08:25.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:25.317006 10513 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:08:25.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:25.317404 10513 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:08:25.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:25.317685 10513 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:08:25.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:25.317972 10513 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:08:25.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:25.318239 10513 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:08:25.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:25.318461 10513 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:36329/jwks.json
08:08:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:25.358727 10513 master_runner.cc:386] Master server non-default flags:
08:08:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.143.212:44461
08:08:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/data
08:08:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/wal
08:08:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:08:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:08:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.143.254:42279,127.9.143.253:32923,127.9.143.252:37215
08:08:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kudu.keytab
08:08:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.143.254
08:08:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:08:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.143.254:42279
08:08:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:08:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:08:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:08:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36329/jwks.json
08:08:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:08:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:08:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:08:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/data/info.pb
08:08:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.143.254
08:08:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42699
08:08:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:08:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:08:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:08:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/logs
08:08:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:08:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:08:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:08:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:08:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
08:08:25.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:08:25.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 07:43:21 UTC on e7f111948823
08:08:25.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11723
08:08:25.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:08:25.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:25.360351 10513 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:08:25.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:25.361568 10513 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:08:25.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:25.373083 10535 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:26.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:26.638422 10542 raft_consensus.cc:493] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:08:26.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:26.639454 10542 raft_consensus.cc:515] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } }
08:08:26.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:26.674273 10542 leader_election.cc:290] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 513a338fca7944dbbf4433e106c2a03f (127.9.143.254:42279), 1d36c7228de444508830a92c33ddaa4a (127.9.143.253:32923)
08:08:26.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:26.681397 10461 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.143.253:32923: connect: Connection refused (error 111)
08:08:26.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:26.693202 10462 leader_election.cc:336] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 513a338fca7944dbbf4433e106c2a03f (127.9.143.254:42279): Network error: Client connection negotiation failed: client connection to 127.9.143.254:42279: connect: Connection refused (error 111)
08:08:26.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:26.697341 10461 leader_election.cc:336] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 1d36c7228de444508830a92c33ddaa4a (127.9.143.253:32923): Network error: Client connection negotiation failed: client connection to 127.9.143.253:32923: connect: Connection refused (error 111)
08:08:26.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:26.698199 10461 leader_election.cc:304] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: ca61a7dcebe145b4a80009b389f47393; no voters: 1d36c7228de444508830a92c33ddaa4a, 513a338fca7944dbbf4433e106c2a03f
08:08:26.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:26.699719 10542 raft_consensus.cc:2749] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
08:08:27.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:26.775058 10534 debug-util.cc:398] Leaking SignalData structure 0x7b0800028d00 after lost signal to thread 10513
08:08:27.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:27.008819 10534 kernel_stack_watchdog.cc:198] Thread 10513 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 398ms:
08:08:27.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:08:27.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:08:27.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:08:27.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:08:27.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:25.375121 10536 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:27.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:27.010001 10513 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.635s user 0.542s sys 0.956s
08:08:27.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:27.010356 10513 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.636s user 0.542s sys 0.956s
08:08:27.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:27.011876 10538 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:27.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:27.014868 10537 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1639 milliseconds
08:08:27.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:27.014940 10513 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:08:27.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:27.016152 10513 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:08:27.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:27.018787 10513 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:08:27.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:27.020177 10513 hybrid_clock.cc:648] HybridClock initialized: now 1777795707020126 us; error 62 us; skew 500 ppm
08:08:27.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:27.021031 10513 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:08:27.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:08:27 dist-test-slave-tr3r krb5kdc[9803](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777795707, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.143.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:08:27.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:27.027091 10513 init.cc:377] Logged in from keytab as kudu/127.9.143.254@KRBTEST.COM (short username kudu)
08:08:27.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:27.032565 10513 webserver.cc:492] Webserver started at http://127.9.143.254:42699/ using document root <none> and password file <none>
08:08:27.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:27.033470 10513 fs_manager.cc:362] Metadata directory not provided
08:08:27.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:27.033723 10513 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:08:27.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:27.053138 10513 fs_manager.cc:714] Time spent opening directory manager: real 0.016s user 0.013s sys 0.004s
08:08:27.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:27.059485 10551 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:08:27.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:27.061033 10513 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.000s sys 0.005s
08:08:27.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:27.061491 10513 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/data,/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/wal
08:08:27.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "513a338fca7944dbbf4433e106c2a03f"
08:08:27.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 08:08:05 on dist-test-slave-tr3r"
08:08:27.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:27.064404 10513 fs_report.cc:389] FS layout report
08:08:27.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:08:27.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/wal
08:08:27.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/wal
08:08:27.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/data/data
08:08:27.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:08:27.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:08:27.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:08:27.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:08:27.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:08:27.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:08:27.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:08:27.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:08:27.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:08:27.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:08:27.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:08:27.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:27.131227 10513 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:08:27.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:27.139910 10513 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:08:27.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:27.140476 10513 kserver.cc:163] Server-wide thread pool size limit: 3276
08:08:27.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:27.225447 10513 rpc_server.cc:307] RPC server started. Bound to: 127.9.143.254:42279
08:08:27.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:27.225451 10602 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.143.254:42279 every 8 connection(s)
08:08:27.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:27.229514 10513 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/data/info.pb
08:08:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:27.238080 9791 external_mini_cluster.cc:1428] Started /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu as pid 10513
08:08:27.238 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:08:27.238 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.9.143.253:32923
08:08:27.239 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
08:08:27.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:27.240605 9791 external_mini_cluster.cc:1366] Running /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu
08:08:27.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu
08:08:27.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/wal
08:08:27.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/data
08:08:27.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:08:27.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:08:27.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:08:27.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:27.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:27.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:27.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/logs
08:08:27.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/data/info.pb
08:08:27.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:27.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:08:27.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:08:27.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:08:27.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:08:27.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:27.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:27.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:27.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:08:27.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:08:27.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:08:27.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:08:27.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.143.253:32923
08:08:27.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.143.253
08:08:27.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37721
08:08:27.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kudu.keytab
08:08:27.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.143.253
08:08:27.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:08:27.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:08:27.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:08:27.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:08:27.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.143.212:44461
08:08:27.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:27.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:27.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:27.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:08:27.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.143.254:42279,127.9.143.253:32923,127.9.143.252:37215
08:08:27.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:08:27.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36329/jwks.json
08:08:27.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:08:27.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:08:27.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:27.242830 10603 sys_catalog.cc:263] Verifying existing consensus state
08:08:27.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:27.250489 10603 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f: Bootstrap starting.
08:08:27.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:27.294287 10603 log.cc:826] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f: Log is configured to *not* fsync() on all Append() calls
08:08:27.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:27.311491 10603 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:08:27.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:27.312747 10603 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f: Bootstrap complete.
08:08:27.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:27.337018 10603 raft_consensus.cc:359] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } }
08:08:27.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:27.339491 10603 raft_consensus.cc:740] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 513a338fca7944dbbf4433e106c2a03f, State: Initialized, Role: FOLLOWER
08:08:27.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:27.340405 10603 consensus_queue.cc:260] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } }
08:08:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:27.343390 10607 sys_catalog.cc:455] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } } }
08:08:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:27.344444 10607 sys_catalog.cc:458] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [sys.catalog]: This master's current role is: FOLLOWER
08:08:27.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:27.345136 10603 sys_catalog.cc:565] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [sys.catalog]: configured and running, proceeding with master startup.
08:08:27.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:27.366125 10618 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
08:08:27.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:08:27 dist-test-slave-tr3r krb5kdc[9803](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777795707, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.143.254@KRBTEST.COM for kudu/127.9.143.254@KRBTEST.COM
08:08:27.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:27.401543 10513 master_runner.cc:186] Error getting master registration for 127.9.143.253:32923: Network error: Client connection negotiation failed: client connection to 127.9.143.253:32923: connect: Connection refused (error 111),
08:08:27.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:27.419821 10513 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:08:27.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:27.642215 10605 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:08:27.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:27.642848 10605 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:08:27.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:27.643538 10605 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:08:27.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:27.679448 10605 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
08:08:27.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:27.679759 10605 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:08:27.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:27.680142 10605 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:08:27.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:27.680500 10605 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:08:27.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:27.680788 10605 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:08:27.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:27.680999 10605 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:08:27.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:27.681177 10605 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:36329/jwks.json
08:08:27.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:27.720819 10605 master_runner.cc:386] Master server non-default flags:
08:08:27.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.143.212:44461
08:08:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/data
08:08:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/wal
08:08:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:08:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:08:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.143.254:42279,127.9.143.253:32923,127.9.143.252:37215
08:08:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kudu.keytab
08:08:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.143.253
08:08:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:08:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.143.253:32923
08:08:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:08:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:08:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:08:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36329/jwks.json
08:08:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:08:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:08:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:08:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/data/info.pb
08:08:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.143.253
08:08:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37721
08:08:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:08:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:08:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:08:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/logs
08:08:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:08:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:08:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:08:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:08:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
08:08:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:08:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 07:43:21 UTC on e7f111948823
08:08:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11723
08:08:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:08:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:27.722065 10605 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:08:27.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:27.723063 10605 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:08:27.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:27.734123 10627 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:28.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:28.387480 10618 catalog_manager.cc:1269] Loaded cluster ID: 3f6f795373ec4e998985e56bad4e1d1d
08:08:28.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:28.387849 10618 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f: loading cluster ID for follower catalog manager: success
08:08:28.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:28.415290 10618 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f: acquiring CA information for follower catalog manager: success
08:08:28.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:28.433943 10618 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:08:28.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:28.701303 10636 raft_consensus.cc:493] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:08:28.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:28.703336 10636 raft_consensus.cc:515] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } }
08:08:28.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:28.722710 10636 leader_election.cc:290] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 513a338fca7944dbbf4433e106c2a03f (127.9.143.254:42279), 1d36c7228de444508830a92c33ddaa4a (127.9.143.253:32923)
08:08:28.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:28.739566 10640 raft_consensus.cc:493] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:08:28.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:28.744354 10640 raft_consensus.cc:515] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } }
08:08:28.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:28.755967 10461 leader_election.cc:336] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 1d36c7228de444508830a92c33ddaa4a (127.9.143.253:32923): Network error: Client connection negotiation failed: client connection to 127.9.143.253:32923: connect: Connection refused (error 111)
08:08:28.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:28.775614 10578 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ca61a7dcebe145b4a80009b389f47393" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "513a338fca7944dbbf4433e106c2a03f" is_pre_election: true
08:08:28.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:28.776531 10578 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ca61a7dcebe145b4a80009b389f47393 in term 2.
08:08:28.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:28.787824 10640 leader_election.cc:290] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 1d36c7228de444508830a92c33ddaa4a (127.9.143.253:32923), ca61a7dcebe145b4a80009b389f47393 (127.9.143.252:37215)
08:08:28.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:28.791245 10462 leader_election.cc:304] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 513a338fca7944dbbf4433e106c2a03f, ca61a7dcebe145b4a80009b389f47393; no voters: 1d36c7228de444508830a92c33ddaa4a
08:08:28.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:28.792198 10636 raft_consensus.cc:2804] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 2 FOLLOWER]: Leader pre-election won for term 3
08:08:28.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:28.792569 10636 raft_consensus.cc:493] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:08:28.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:28.792879 10636 raft_consensus.cc:3060] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 2 FOLLOWER]: Advancing to term 3
08:08:28.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:28.804142 10636 raft_consensus.cc:515] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } }
08:08:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:28.807149 10578 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ca61a7dcebe145b4a80009b389f47393" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "513a338fca7944dbbf4433e106c2a03f"
08:08:28.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:28.807786 10578 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [term 2 FOLLOWER]: Advancing to term 3
08:08:28.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:28.814539 10636 leader_election.cc:290] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [CANDIDATE]: Term 3 election: Requested vote from peers 513a338fca7944dbbf4433e106c2a03f (127.9.143.254:42279), 1d36c7228de444508830a92c33ddaa4a (127.9.143.253:32923)
08:08:28.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:28.818672 10461 leader_election.cc:336] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 1d36c7228de444508830a92c33ddaa4a (127.9.143.253:32923): Network error: Client connection negotiation failed: client connection to 127.9.143.253:32923: connect: Connection refused (error 111)
08:08:28.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:28.819691 10578 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ca61a7dcebe145b4a80009b389f47393 in term 3.
08:08:28.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:28.821229 10462 leader_election.cc:304] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 513a338fca7944dbbf4433e106c2a03f, ca61a7dcebe145b4a80009b389f47393; no voters: 1d36c7228de444508830a92c33ddaa4a
08:08:28.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:28.822371 10636 raft_consensus.cc:2804] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 3 FOLLOWER]: Leader election won for term 3
08:08:28.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:28.827948 10553 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.143.253:32923: connect: Connection refused (error 111)
08:08:28.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:28.842312 10636 raft_consensus.cc:697] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 3 LEADER]: Becoming Leader. State: Replica: ca61a7dcebe145b4a80009b389f47393, State: Running, Role: LEADER
08:08:28.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:28.845242 10636 consensus_queue.cc:237] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } }
08:08:28.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:28.851675 10553 leader_election.cc:336] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 1d36c7228de444508830a92c33ddaa4a (127.9.143.253:32923): Network error: Client connection negotiation failed: client connection to 127.9.143.253:32923: connect: Connection refused (error 111)
08:08:28.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:28.863914 10486 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "513a338fca7944dbbf4433e106c2a03f" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "ca61a7dcebe145b4a80009b389f47393" is_pre_election: true
08:08:28.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:28.866591 10555 leader_election.cc:304] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 513a338fca7944dbbf4433e106c2a03f; no voters: 1d36c7228de444508830a92c33ddaa4a, ca61a7dcebe145b4a80009b389f47393
08:08:28.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:28.867507 10640 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [term 3 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
08:08:28.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:28.872437 10645 sys_catalog.cc:455] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [sys.catalog]: SysCatalogTable state changed. Reason: New leader ca61a7dcebe145b4a80009b389f47393. Latest consensus state: current_term: 3 leader_uuid: "ca61a7dcebe145b4a80009b389f47393" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } } }
08:08:28.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:28.873502 10645 sys_catalog.cc:458] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [sys.catalog]: This master's current role is: LEADER
08:08:28.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:28.876984 10648 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
08:08:28.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:28.896279 10648 catalog_manager.cc:1494] Initializing Kudu cluster ID...
08:08:28.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:28.901777 10648 catalog_manager.cc:1269] Loaded cluster ID: 3f6f795373ec4e998985e56bad4e1d1d
08:08:28.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:28.907065 10648 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
08:08:28.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:28.913121 10648 catalog_manager.cc:1514] Loading token signing keys...
08:08:28.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:28.942234 10648 catalog_manager.cc:6055] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393: Loaded TSK: 0
08:08:28.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:28.948611 10648 catalog_manager.cc:1524] Initializing in-progress tserver states...
08:08:29.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.240723 10578 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [term 3 FOLLOWER]: Refusing update from remote peer ca61a7dcebe145b4a80009b389f47393: Log matching property violated. Preceding OpId in replica: term: 2 index: 5. Preceding OpId from leader: term: 3 index: 6. (index mismatch)
08:08:29.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.242537 10645 consensus_queue.cc:1048] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [LEADER]: Connected to new peer: Peer: permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
08:08:29.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.266973 10640 sys_catalog.cc:455] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [sys.catalog]: SysCatalogTable state changed. Reason: New leader ca61a7dcebe145b4a80009b389f47393. Latest consensus state: current_term: 3 leader_uuid: "ca61a7dcebe145b4a80009b389f47393" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } } }
08:08:29.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.267710 10640 sys_catalog.cc:458] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [sys.catalog]: This master's current role is: FOLLOWER
08:08:29.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:29.136616 10626 debug-util.cc:398] Leaking SignalData structure 0x7b0800028d00 after lost signal to thread 10605
08:08:29.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:29.289600 10605 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.554s user 0.603s sys 0.921s
08:08:29.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:29.289981 10605 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.555s user 0.603s sys 0.921s
08:08:29.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:27.735435 10628 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:29.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:29.291934 10630 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:29.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.293054 10640 sys_catalog.cc:455] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "ca61a7dcebe145b4a80009b389f47393" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } } }
08:08:29.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.293788 10640 sys_catalog.cc:458] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [sys.catalog]: This master's current role is: FOLLOWER
08:08:29.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.296584 10645 sys_catalog.cc:455] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "ca61a7dcebe145b4a80009b389f47393" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } } }
08:08:29.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.297257 10645 sys_catalog.cc:458] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [sys.catalog]: This master's current role is: LEADER
08:08:29.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:29.299825 10461 consensus_peers.cc:597] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 -> Peer 1d36c7228de444508830a92c33ddaa4a (127.9.143.253:32923): Couldn't send request to peer 1d36c7228de444508830a92c33ddaa4a. Status: Network error: Client connection negotiation failed: client connection to 127.9.143.253:32923: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:08:29.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.301139 10605 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:08:29.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:29.301817 10629 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1555 milliseconds
08:08:29.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.303840 10605 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:08:29.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.306954 10605 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:08:29.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.308483 10605 hybrid_clock.cc:648] HybridClock initialized: now 1777795709308388 us; error 95 us; skew 500 ppm
08:08:29.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.309556 10605 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:08:29.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:08:29 dist-test-slave-tr3r krb5kdc[9803](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777795709, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.143.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:08:29.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.316766 10605 init.cc:377] Logged in from keytab as kudu/127.9.143.253@KRBTEST.COM (short username kudu)
08:08:29.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.321676 10605 webserver.cc:492] Webserver started at http://127.9.143.253:37721/ using document root <none> and password file <none>
08:08:29.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.322616 10605 fs_manager.cc:362] Metadata directory not provided
08:08:29.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.322816 10605 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:08:29.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.331938 10605 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.003s sys 0.002s
08:08:29.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.336786 10657 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:08:29.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.338287 10605 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s
08:08:29.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.338678 10605 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/data,/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/wal
08:08:29.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1d36c7228de444508830a92c33ddaa4a"
08:08:29.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 08:08:08 on dist-test-slave-tr3r"
08:08:29.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.340672 10605 fs_report.cc:389] FS layout report
08:08:29.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:08:29.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/wal
08:08:29.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/wal
08:08:29.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/data/data
08:08:29.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:08:29.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:08:29.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:08:29.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:08:29.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:08:29.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:08:29.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:08:29.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:08:29.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:08:29.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:08:29.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:08:29.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.393399 10605 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:08:29.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.399649 10605 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:08:29.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.400144 10605 kserver.cc:163] Server-wide thread pool size limit: 3276
08:08:29.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.495505 10605 rpc_server.cc:307] RPC server started. Bound to: 127.9.143.253:32923
08:08:29.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.495604 10708 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.143.253:32923 every 8 connection(s)
08:08:29.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.499161 10605 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/data/info.pb
08:08:29.501 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:08:29.502 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.9.143.252:37215(127.9.143.252:37215)] connecting to peer
08:08:29.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.500867 9791 external_mini_cluster.cc:1428] Started /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu as pid 10605
08:08:29.504 [DEBUG - kudu-nio-3] (Connection.java:757) [peer master-127.9.143.252:37215(127.9.143.252:37215)] Successfully connected to peer
08:08:29.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.515029 10709 sys_catalog.cc:263] Verifying existing consensus state
08:08:29.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.522872 10709 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a: Bootstrap starting.
08:08:29.532 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0x3343275b, L:/127.0.0.1:57486 - R:/127.9.143.252:37215] using TOKEN/null
08:08:29.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.571646 10709 log.cc:826] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a: Log is configured to *not* fsync() on all Append() calls
08:08:29.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.588992 10709 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:08:29.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.589864 10709 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a: Bootstrap complete.
08:08:29.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.625396 10709 raft_consensus.cc:359] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } }
08:08:29.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.628787 10709 raft_consensus.cc:740] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1d36c7228de444508830a92c33ddaa4a, State: Initialized, Role: FOLLOWER
08:08:29.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.629841 10709 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } }
08:08:29.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.632683 10714 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } } }
08:08:29.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.633484 10714 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a [sys.catalog]: This master's current role is: FOLLOWER
08:08:29.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.634479 10709 sys_catalog.cc:565] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a [sys.catalog]: configured and running, proceeding with master startup.
08:08:29.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.676754 10725 catalog_manager.cc:1269] Loaded cluster ID: 3f6f795373ec4e998985e56bad4e1d1d
08:08:29.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.680476 10725 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a: loading cluster ID for follower catalog manager: success
08:08:29.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:08:29 dist-test-slave-tr3r krb5kdc[9803](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777795709, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.143.253@KRBTEST.COM for kudu/127.9.143.254@KRBTEST.COM
08:08:29.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.689355 10725 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a: acquiring CA information for follower catalog manager: success
08:08:29.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.694705 10725 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:08:29.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.740940 10684 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a [term 2 FOLLOWER]: Advancing to term 3
08:08:29.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.760936 10732 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "ca61a7dcebe145b4a80009b389f47393" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } } }
08:08:29.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.762281 10732 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a [sys.catalog]: This master's current role is: FOLLOWER
08:08:29.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.768250 10714 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a [sys.catalog]: SysCatalogTable state changed. Reason: New leader ca61a7dcebe145b4a80009b389f47393. Latest consensus state: current_term: 3 leader_uuid: "ca61a7dcebe145b4a80009b389f47393" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } } }
08:08:29.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:29.768989 10714 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a [sys.catalog]: This master's current role is: FOLLOWER
08:08:30.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:30.211702 10636 sys_catalog.cc:455] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "ca61a7dcebe145b4a80009b389f47393" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } } }
08:08:30.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:30.212508 10636 sys_catalog.cc:458] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [sys.catalog]: This master's current role is: LEADER
08:08:32.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:32.273366 9216 debug-util.cc:398] Leaking SignalData structure 0x7b08000b1180 after lost signal to thread 9151
08:08:32.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:32.273973 9216 debug-util.cc:398] Leaking SignalData structure 0x7b08000a8840 after lost signal to thread 9219
08:08:32.585 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.9.143.252:37215
08:08:32.585 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
08:08:32.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:32.586186 9791 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu with pid 10442
08:08:32.618 [DEBUG - kudu-nio-3] (Connection.java:239) [peer master-127.9.143.252:37215(127.9.143.252:37215)] handling channelInactive
08:08:32.618 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.9.143.252:37215(127.9.143.252:37215)] cleaning up while in state READY due to: connection closed
08:08:32.620 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:08:32.620 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.9.143.254:42279
08:08:32.620 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
08:08:32.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:32.620635 9791 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu with pid 10513
08:08:32.649 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:08:32.649 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.9.143.253:32923
08:08:32.649 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
08:08:32.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:32.650032 9791 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu with pid 10605
08:08:32.678 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:08:32.678 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.9.143.252:37215
08:08:32.678 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
08:08:32.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:32.680080 9791 external_mini_cluster.cc:1366] Running /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu
08:08:32.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu
08:08:32.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/wal
08:08:32.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/data
08:08:32.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:08:32.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:08:32.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:08:32.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:32.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:32.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:32.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/logs
08:08:32.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/data/info.pb
08:08:32.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:32.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:08:32.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:08:32.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:08:32.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:08:32.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:32.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:32.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:32.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:08:32.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:08:32.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:08:32.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:08:32.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.143.252:37215
08:08:32.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.143.252
08:08:32.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45863
08:08:32.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kudu.keytab
08:08:32.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.143.252
08:08:32.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:08:32.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:08:32.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:08:32.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:08:32.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.143.212:44461
08:08:32.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:32.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:32.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:32.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:08:32.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.143.254:42279,127.9.143.253:32923,127.9.143.252:37215
08:08:32.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:08:32.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36329/jwks.json
08:08:32.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:08:32.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:08:33.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:33.068691 10742 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:08:33.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:33.069234 10742 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:08:33.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:33.069904 10742 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:08:33.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:33.105186 10742 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
08:08:33.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:33.105427 10742 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:08:33.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:33.105803 10742 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:08:33.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:33.106135 10742 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:08:33.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:33.106415 10742 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:08:33.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:33.106621 10742 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:08:33.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:33.106771 10742 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:36329/jwks.json
08:08:33.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:33.147490 10742 master_runner.cc:386] Master server non-default flags:
08:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.143.212:44461
08:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/data
08:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/wal
08:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.143.254:42279,127.9.143.253:32923,127.9.143.252:37215
08:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kudu.keytab
08:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.143.252
08:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.143.252:37215
08:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36329/jwks.json
08:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/data/info.pb
08:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.143.252
08:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45863
08:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/logs
08:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
08:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 07:43:21 UTC on e7f111948823
08:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11723
08:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:33.148759 10742 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:08:33.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:33.149817 10742 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:08:33.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:33.162551 10748 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:33.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:33.799115 9495 debug-util.cc:398] Leaking SignalData structure 0x7b08000adb00 after lost signal to thread 9365
08:08:33.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:33.800199 9495 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad440 after lost signal to thread 9498
08:08:35.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:34.563917 10747 debug-util.cc:398] Leaking SignalData structure 0x7b0800028d00 after lost signal to thread 10742
08:08:35.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:35.225947 10747 kernel_stack_watchdog.cc:198] Thread 10742 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 398ms:
08:08:35.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:08:35.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:08:35.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:08:35.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:08:35.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:35.226855 10742 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.063s user 0.742s sys 1.063s
08:08:35.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:35.227212 10742 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.064s user 0.742s sys 1.063s
08:08:35.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:33.163772 10749 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:35.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:35.229327 10751 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:35.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:35.232998 10750 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2068 milliseconds
08:08:35.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:35.233018 10742 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:08:35.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:35.234366 10742 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:08:35.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:35.237041 10742 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:08:35.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:35.238448 10742 hybrid_clock.cc:648] HybridClock initialized: now 1777795715238402 us; error 57 us; skew 500 ppm
08:08:35.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:35.239614 10742 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:08:35.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:08:35 dist-test-slave-tr3r krb5kdc[9803](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777795715, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.143.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:08:35.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:35.247061 10742 init.cc:377] Logged in from keytab as kudu/127.9.143.252@KRBTEST.COM (short username kudu)
08:08:35.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:35.252936 10742 webserver.cc:492] Webserver started at http://127.9.143.252:45863/ using document root <none> and password file <none>
08:08:35.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:35.254181 10742 fs_manager.cc:362] Metadata directory not provided
08:08:35.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:35.254452 10742 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:08:35.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:35.265121 10742 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.003s sys 0.003s
08:08:35.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:35.271027 10760 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:08:35.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:35.272222 10742 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
08:08:35.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:35.272568 10742 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/data,/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/wal
08:08:35.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ca61a7dcebe145b4a80009b389f47393"
08:08:35.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 08:08:10 on dist-test-slave-tr3r"
08:08:35.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:35.274556 10742 fs_report.cc:389] FS layout report
08:08:35.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:08:35.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/wal
08:08:35.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/wal
08:08:35.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/data/data
08:08:35.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:08:35.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:08:35.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:08:35.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:08:35.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:08:35.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:08:35.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:08:35.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:08:35.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:08:35.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:08:35.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:08:35.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:35.349424 10742 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:08:35.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:35.358871 10742 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:08:35.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:35.359496 10742 kserver.cc:163] Server-wide thread pool size limit: 3276
08:08:35.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:35.448057 10742 rpc_server.cc:307] RPC server started. Bound to: 127.9.143.252:37215
08:08:35.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:35.448197 10811 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.143.252:37215 every 8 connection(s)
08:08:35.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:35.451087 10742 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-2/data/info.pb
08:08:35.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:35.455169 9791 external_mini_cluster.cc:1428] Started /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu as pid 10742
08:08:35.455 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:08:35.455 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.9.143.254:42279
08:08:35.456 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
08:08:35.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:35.457615 9791 external_mini_cluster.cc:1366] Running /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu
08:08:35.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu
08:08:35.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/wal
08:08:35.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/data
08:08:35.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:08:35.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:08:35.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:08:35.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:35.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:35.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:35.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/logs
08:08:35.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/data/info.pb
08:08:35.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:35.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:08:35.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:08:35.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:08:35.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:08:35.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:35.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:35.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:35.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:08:35.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:08:35.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:08:35.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:08:35.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.143.254:42279
08:08:35.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.143.254
08:08:35.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42699
08:08:35.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kudu.keytab
08:08:35.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.143.254
08:08:35.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:08:35.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:08:35.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:08:35.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:08:35.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.143.212:44461
08:08:35.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:35.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:35.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:35.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:08:35.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.143.254:42279,127.9.143.253:32923,127.9.143.252:37215
08:08:35.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:08:35.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36329/jwks.json
08:08:35.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:08:35.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:08:35.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:35.466413 10812 sys_catalog.cc:263] Verifying existing consensus state
08:08:35.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:35.474357 9216 debug-util.cc:398] Leaking SignalData structure 0x7b080006fda0 after lost signal to thread 9151
08:08:35.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:35.474416 10812 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393: Bootstrap starting.
08:08:35.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:35.474954 9216 debug-util.cc:398] Leaking SignalData structure 0x7b0800070280 after lost signal to thread 9219
08:08:35.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:35.528736 10812 log.cc:826] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393: Log is configured to *not* fsync() on all Append() calls
08:08:35.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:35.570623 10812 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:08:35.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:35.571587 10812 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393: Bootstrap complete.
08:08:35.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:35.605777 10812 raft_consensus.cc:359] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } }
08:08:35.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:35.609097 10812 raft_consensus.cc:740] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: ca61a7dcebe145b4a80009b389f47393, State: Initialized, Role: FOLLOWER
08:08:35.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:35.610145 10812 consensus_queue.cc:260] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } }
08:08:35.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:35.613181 10816 sys_catalog.cc:455] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } } }
08:08:35.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:35.614065 10816 sys_catalog.cc:458] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [sys.catalog]: This master's current role is: FOLLOWER
08:08:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:35.614917 10812 sys_catalog.cc:565] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [sys.catalog]: configured and running, proceeding with master startup.
08:08:35.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:35.657542 10827 catalog_manager.cc:1269] Loaded cluster ID: 3f6f795373ec4e998985e56bad4e1d1d
08:08:35.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:35.657948 10827 catalog_manager.cc:1562] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393: loading cluster ID for follower catalog manager: success
08:08:35.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:35.674714 10827 catalog_manager.cc:1584] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393: acquiring CA information for follower catalog manager: success
08:08:35.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:35.693737 10827 catalog_manager.cc:1612] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:08:35.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:35.658749 10742 master_runner.cc:186] Error getting master registration for 127.9.143.254:42279: Network error: Client connection negotiation failed: client connection to 127.9.143.254:42279: connect: Connection refused (error 111),
08:08:35.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:35.699149 10742 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:08:35.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:35.891140 10814 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:08:35.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:35.891830 10814 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:08:35.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:35.892585 10814 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:08:35.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:35.928867 10814 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
08:08:35.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:35.929133 10814 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:08:35.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:35.929514 10814 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:08:35.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:35.929843 10814 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:08:35.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:35.930171 10814 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:08:35.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:35.930486 10814 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:08:35.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:35.930727 10814 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:36329/jwks.json
08:08:35.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:35.974148 10814 master_runner.cc:386] Master server non-default flags:
08:08:35.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:35.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.143.212:44461
08:08:35.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:35.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:35.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/data
08:08:35.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/wal
08:08:35.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:08:35.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:08:35.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.143.254:42279,127.9.143.253:32923,127.9.143.252:37215
08:08:35.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kudu.keytab
08:08:35.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.143.254
08:08:35.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:35.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:35.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:08:35.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.143.254:42279
08:08:35.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:08:35.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:08:35.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:08:35.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36329/jwks.json
08:08:35.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:08:35.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:08:35.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:08:35.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:35.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:35.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/data/info.pb
08:08:35.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.143.254
08:08:35.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42699
08:08:35.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:08:35.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:35.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:08:35.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:08:35.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:35.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/logs
08:08:35.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:35.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:08:35.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:08:35.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:08:35.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:08:35.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
08:08:35.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:08:35.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 07:43:21 UTC on e7f111948823
08:08:35.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11723
08:08:35.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:08:35.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:35.975842 10814 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:08:35.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:35.977047 10814 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:08:35.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:35.989363 10835 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:36.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:36.796434 10841 raft_consensus.cc:493] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:08:36.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:36.797253 10841 raft_consensus.cc:515] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } }
08:08:36.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:36.818706 10841 leader_election.cc:290] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 513a338fca7944dbbf4433e106c2a03f (127.9.143.254:42279), 1d36c7228de444508830a92c33ddaa4a (127.9.143.253:32923)
08:08:36.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:36.827139 10762 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.143.253:32923: connect: Connection refused (error 111)
08:08:36.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:36.847640 10763 leader_election.cc:336] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 513a338fca7944dbbf4433e106c2a03f (127.9.143.254:42279): Network error: Client connection negotiation failed: client connection to 127.9.143.254:42279: connect: Connection refused (error 111)
08:08:36.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:36.850708 10762 leader_election.cc:336] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 1d36c7228de444508830a92c33ddaa4a (127.9.143.253:32923): Network error: Client connection negotiation failed: client connection to 127.9.143.253:32923: connect: Connection refused (error 111)
08:08:36.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:36.851164 10762 leader_election.cc:304] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: ca61a7dcebe145b4a80009b389f47393; no voters: 1d36c7228de444508830a92c33ddaa4a, 513a338fca7944dbbf4433e106c2a03f
08:08:36.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:36.851953 10841 raft_consensus.cc:2749] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
08:08:37.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:37.391387 10834 debug-util.cc:398] Leaking SignalData structure 0x7b0800028d00 after lost signal to thread 10814
08:08:37.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:37.693475 10814 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.705s user 0.702s sys 0.939s
08:08:37.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:37.694000 10814 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.705s user 0.703s sys 0.939s
08:08:37.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:35.991080 10836 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:37.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:37.705394 10814 server_base.cc:1061] running on GCE node
08:08:37.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:37.705900 10846 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:37.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:37.707348 10814 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:08:37.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:37.711201 10814 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:08:37.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:37.712800 10814 hybrid_clock.cc:648] HybridClock initialized: now 1777795717712714 us; error 84 us; skew 500 ppm
08:08:37.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:37.713861 10814 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:08:37.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:08:37 dist-test-slave-tr3r krb5kdc[9803](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777795717, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.143.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:08:37.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:37.720458 10814 init.cc:377] Logged in from keytab as kudu/127.9.143.254@KRBTEST.COM (short username kudu)
08:08:37.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:37.725894 10814 webserver.cc:492] Webserver started at http://127.9.143.254:42699/ using document root <none> and password file <none>
08:08:37.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:37.726848 10814 fs_manager.cc:362] Metadata directory not provided
08:08:37.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:37.727111 10814 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:08:37.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:37.735746 10814 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.004s sys 0.000s
08:08:37.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:37.740844 10853 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:08:37.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:37.742177 10814 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
08:08:37.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:37.742601 10814 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/data,/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/wal
08:08:37.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "513a338fca7944dbbf4433e106c2a03f"
08:08:37.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 08:08:05 on dist-test-slave-tr3r"
08:08:37.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:37.745405 10814 fs_report.cc:389] FS layout report
08:08:37.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:08:37.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/wal
08:08:37.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/wal
08:08:37.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/data/data
08:08:37.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:08:37.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:08:37.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:08:37.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:08:37.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:08:37.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:08:37.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:08:37.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:08:37.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:08:37.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:08:37.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:08:37.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:37.810879 10814 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:08:37.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:37.818607 10814 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:08:37.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:37.819156 10814 kserver.cc:163] Server-wide thread pool size limit: 3276
08:08:37.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:37.903281 10814 rpc_server.cc:307] RPC server started. Bound to: 127.9.143.254:42279
08:08:37.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:37.903322 10904 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.143.254:42279 every 8 connection(s)
08:08:37.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:37.907261 10814 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-0/data/info.pb
08:08:37.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:37.910951 9791 external_mini_cluster.cc:1428] Started /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu as pid 10814
08:08:37.911 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:08:37.911 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.9.143.253:32923
08:08:37.911 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
08:08:37.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:37.913625 9791 external_mini_cluster.cc:1366] Running /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu
08:08:37.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu
08:08:37.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/wal
08:08:37.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/data
08:08:37.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:08:37.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:08:37.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:08:37.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:37.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:37.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:37.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/logs
08:08:37.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/data/info.pb
08:08:37.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:37.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:08:37.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:08:37.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:08:37.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:08:37.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:37.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:37.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:37.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:08:37.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:08:37.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:08:37.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:08:37.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.143.253:32923
08:08:37.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.143.253
08:08:37.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37721
08:08:37.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kudu.keytab
08:08:37.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.143.253
08:08:37.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:08:37.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:08:37.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:08:37.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:08:37.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.143.212:44461
08:08:37.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:37.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:37.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:37.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:08:37.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.143.254:42279,127.9.143.253:32923,127.9.143.252:37215
08:08:37.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:08:37.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36329/jwks.json
08:08:37.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:08:37.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:08:37.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:37.920293 10905 sys_catalog.cc:263] Verifying existing consensus state
08:08:37.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:37.926115 10905 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f: Bootstrap starting.
08:08:37.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:37.969158 10905 log.cc:826] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f: Log is configured to *not* fsync() on all Append() calls
08:08:37.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:37.986872 10905 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:08:37.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:37.987707 10905 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f: Bootstrap complete.
08:08:38.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:38.013104 10905 raft_consensus.cc:359] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } }
08:08:38.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:38.016326 10905 raft_consensus.cc:740] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 513a338fca7944dbbf4433e106c2a03f, State: Initialized, Role: FOLLOWER
08:08:38.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:38.017297 10905 consensus_queue.cc:260] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } }
08:08:38.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:38.020328 10909 sys_catalog.cc:455] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } } }
08:08:38.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:38.021952 10909 sys_catalog.cc:458] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [sys.catalog]: This master's current role is: FOLLOWER
08:08:38.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:38.022056 10905 sys_catalog.cc:565] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [sys.catalog]: configured and running, proceeding with master startup.
08:08:38.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:38.070175 10920 catalog_manager.cc:1269] Loaded cluster ID: 3f6f795373ec4e998985e56bad4e1d1d
08:08:38.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:38.070462 10920 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f: loading cluster ID for follower catalog manager: success
08:08:38.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:38.085901 10920 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f: acquiring CA information for follower catalog manager: success
08:08:38.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:08:38 dist-test-slave-tr3r krb5kdc[9803](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777795717, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.143.254@KRBTEST.COM for kudu/127.9.143.254@KRBTEST.COM
08:08:38.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:38.095139 10920 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:08:38.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:38.118978 10814 master_runner.cc:186] Error getting master registration for 127.9.143.253:32923: Network error: Client connection negotiation failed: client connection to 127.9.143.253:32923: connect: Connection refused (error 111),
08:08:38.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:38.145385 10814 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:08:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:38.317703 10907 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:08:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:38.318404 10907 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:08:38.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:38.319092 10907 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:08:38.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:38.359172 10907 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
08:08:38.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:38.359449 10907 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:08:38.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:38.359807 10907 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:08:38.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:38.360108 10907 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:08:38.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:38.360387 10907 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:08:38.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:38.360684 10907 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:08:38.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:38.360898 10907 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:36329/jwks.json
08:08:38.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:38.401870 10907 master_runner.cc:386] Master server non-default flags:
08:08:38.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:38.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.143.212:44461
08:08:38.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:38.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:38.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/data
08:08:38.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/wal
08:08:38.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:08:38.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:08:38.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.143.254:42279,127.9.143.253:32923,127.9.143.252:37215
08:08:38.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/kudu.keytab
08:08:38.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.143.253
08:08:38.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:38.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:38.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:08:38.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.143.253:32923
08:08:38.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:08:38.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:08:38.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:08:38.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36329/jwks.json
08:08:38.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:08:38.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:08:38.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:08:38.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:38.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:38.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/data/info.pb
08:08:38.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.143.253
08:08:38.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37721
08:08:38.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:08:38.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:38.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:08:38.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:08:38.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:38.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/logs
08:08:38.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:38.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:08:38.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:08:38.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:08:38.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:08:38.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
08:08:38.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:08:38.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 07:43:21 UTC on e7f111948823
08:08:38.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11723
08:08:38.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:08:38.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:38.403578 10907 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:08:38.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:38.404695 10907 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:08:38.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:38.417277 10930 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:38.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:38.514106 10936 raft_consensus.cc:493] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:08:38.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:38.515111 10936 raft_consensus.cc:515] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } }
08:08:38.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:38.527482 10936 leader_election.cc:290] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 513a338fca7944dbbf4433e106c2a03f (127.9.143.254:42279), 1d36c7228de444508830a92c33ddaa4a (127.9.143.253:32923)
08:08:38.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:38.538108 10880 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ca61a7dcebe145b4a80009b389f47393" candidate_term: 4 candidate_status { last_received { term: 3 index: 6 } } ignore_live_leader: false dest_uuid: "513a338fca7944dbbf4433e106c2a03f" is_pre_election: true
08:08:38.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:38.539436 10880 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ca61a7dcebe145b4a80009b389f47393 in term 3.
08:08:38.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:38.541162 10763 leader_election.cc:304] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 513a338fca7944dbbf4433e106c2a03f, ca61a7dcebe145b4a80009b389f47393; no voters:
08:08:38.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:38.542490 10936 raft_consensus.cc:2804] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 3 FOLLOWER]: Leader pre-election won for term 4
08:08:38.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:38.542858 10936 raft_consensus.cc:493] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 3 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:08:38.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:38.543188 10936 raft_consensus.cc:3060] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 3 FOLLOWER]: Advancing to term 4
08:08:38.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:38.554865 10936 raft_consensus.cc:515] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } }
08:08:38.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:38.557049 10762 leader_election.cc:336] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 1d36c7228de444508830a92c33ddaa4a (127.9.143.253:32923): Network error: Client connection negotiation failed: client connection to 127.9.143.253:32923: connect: Connection refused (error 111)
08:08:38.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:38.558847 10880 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ca61a7dcebe145b4a80009b389f47393" candidate_term: 4 candidate_status { last_received { term: 3 index: 6 } } ignore_live_leader: false dest_uuid: "513a338fca7944dbbf4433e106c2a03f"
08:08:38.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:38.559406 10880 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [term 3 FOLLOWER]: Advancing to term 4
08:08:38.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:38.563880 10762 leader_election.cc:336] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 1d36c7228de444508830a92c33ddaa4a (127.9.143.253:32923): Network error: Client connection negotiation failed: client connection to 127.9.143.253:32923: connect: Connection refused (error 111)
08:08:38.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:38.558422 10936 leader_election.cc:290] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [CANDIDATE]: Term 4 election: Requested vote from peers 513a338fca7944dbbf4433e106c2a03f (127.9.143.254:42279), 1d36c7228de444508830a92c33ddaa4a (127.9.143.253:32923)
08:08:38.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:38.568867 10880 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ca61a7dcebe145b4a80009b389f47393 in term 4.
08:08:38.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:38.570043 10763 leader_election.cc:304] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 513a338fca7944dbbf4433e106c2a03f, ca61a7dcebe145b4a80009b389f47393; no voters: 1d36c7228de444508830a92c33ddaa4a
08:08:38.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:38.571604 10936 raft_consensus.cc:2804] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 4 FOLLOWER]: Leader election won for term 4
08:08:38.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:38.586097 10936 raft_consensus.cc:697] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [term 4 LEADER]: Becoming Leader. State: Replica: ca61a7dcebe145b4a80009b389f47393, State: Running, Role: LEADER
08:08:38.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:38.587316 10936 consensus_queue.cc:237] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } }
08:08:38.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:38.609243 10941 sys_catalog.cc:455] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [sys.catalog]: SysCatalogTable state changed. Reason: New leader ca61a7dcebe145b4a80009b389f47393. Latest consensus state: current_term: 4 leader_uuid: "ca61a7dcebe145b4a80009b389f47393" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } } }
08:08:38.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:38.610877 10941 sys_catalog.cc:458] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [sys.catalog]: This master's current role is: LEADER
08:08:38.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:38.615589 10943 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
08:08:38.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:38.623486 10943 catalog_manager.cc:1494] Initializing Kudu cluster ID...
08:08:38.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:38.627462 10943 catalog_manager.cc:1269] Loaded cluster ID: 3f6f795373ec4e998985e56bad4e1d1d
08:08:38.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:38.627787 10943 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
08:08:38.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:38.632709 10943 catalog_manager.cc:1514] Loading token signing keys...
08:08:38.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:38.636711 10943 catalog_manager.cc:6055] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393: Loaded TSK: 0
08:08:38.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:38.638563 10943 catalog_manager.cc:1524] Initializing in-progress tserver states...
08:08:39.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:39.114146 10762 consensus_peers.cc:597] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 -> Peer 1d36c7228de444508830a92c33ddaa4a (127.9.143.253:32923): Couldn't send request to peer 1d36c7228de444508830a92c33ddaa4a. Status: Network error: Client connection negotiation failed: client connection to 127.9.143.253:32923: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:08:39.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:39.120004 10880 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [term 4 FOLLOWER]: Refusing update from remote peer ca61a7dcebe145b4a80009b389f47393: Log matching property violated. Preceding OpId in replica: term: 3 index: 6. Preceding OpId from leader: term: 4 index: 7. (index mismatch)
08:08:39.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:39.122115 10941 consensus_queue.cc:1048] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [LEADER]: Connected to new peer: Peer: permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.001s
08:08:39.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:39.137687 10947 sys_catalog.cc:455] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [sys.catalog]: SysCatalogTable state changed. Reason: New leader ca61a7dcebe145b4a80009b389f47393. Latest consensus state: current_term: 4 leader_uuid: "ca61a7dcebe145b4a80009b389f47393" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } } }
08:08:39.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:39.138656 10947 sys_catalog.cc:458] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [sys.catalog]: This master's current role is: FOLLOWER
08:08:39.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:39.148589 10947 sys_catalog.cc:455] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "ca61a7dcebe145b4a80009b389f47393" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } } }
08:08:39.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:39.149416 10947 sys_catalog.cc:458] T 00000000000000000000000000000000 P 513a338fca7944dbbf4433e106c2a03f [sys.catalog]: This master's current role is: FOLLOWER
08:08:39.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:39.152091 10941 sys_catalog.cc:455] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "ca61a7dcebe145b4a80009b389f47393" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "513a338fca7944dbbf4433e106c2a03f" member_type: VOTER last_known_addr { host: "127.9.143.254" port: 42279 } } peers { permanent_uuid: "1d36c7228de444508830a92c33ddaa4a" member_type: VOTER last_known_addr { host: "127.9.143.253" port: 32923 } } peers { permanent_uuid: "ca61a7dcebe145b4a80009b389f47393" member_type: VOTER last_known_addr { host: "127.9.143.252" port: 37215 } } }
08:08:39.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:39.153637 10941 sys_catalog.cc:458] T 00000000000000000000000000000000 P ca61a7dcebe145b4a80009b389f47393 [sys.catalog]: This master's current role is: LEADER
08:08:39.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:38.419620 10931 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:39.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:38.421545 10933 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:39.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:39.652520 10932 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
08:08:39.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:39.652635 10907 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:08:39.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:39.654007 10907 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:08:39.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:39.656852 10907 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:08:39.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:39.658231 10907 hybrid_clock.cc:648] HybridClock initialized: now 1777795719658180 us; error 59 us; skew 500 ppm
08:08:39.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:39.659431 10907 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:08:39.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:08:39 dist-test-slave-tr3r krb5kdc[9803](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777795719, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.143.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:08:39.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:39.666079 10907 init.cc:377] Logged in from keytab as kudu/127.9.143.253@KRBTEST.COM (short username kudu)
08:08:39.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:39.671361 10907 webserver.cc:492] Webserver started at http://127.9.143.253:37721/ using document root <none> and password file <none>
08:08:39.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:39.672293 10907 fs_manager.cc:362] Metadata directory not provided
08:08:39.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:39.672549 10907 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:08:39.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:39.681198 10907 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.006s sys 0.000s
08:08:39.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:39.687117 10959 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:08:39.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:39.688376 10907 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.000s sys 0.005s
08:08:39.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:39.688762 10907 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/data,/tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/wal
08:08:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1d36c7228de444508830a92c33ddaa4a"
08:08:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 08:08:08 on dist-test-slave-tr3r"
08:08:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:39.690929 10907 fs_report.cc:389] FS layout report
08:08:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:08:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/wal
08:08:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/wal
08:08:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/data/data
08:08:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:08:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:08:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:08:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:08:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:08:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:08:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:08:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:08:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:08:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:08:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:08:39.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:39.742704 10907 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:08:39.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:39.748382 10907 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:08:39.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:39.748783 10907 kserver.cc:163] Server-wide thread pool size limit: 3276
08:08:39.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:39.840788 10907 rpc_server.cc:307] RPC server started. Bound to: 127.9.143.253:32923
08:08:39.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:39.841334 11012 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.143.253:32923 every 8 connection(s)
08:08:39.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:39.844525 10907 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/master-1/data/info.pb
08:08:39.846 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:08:39.847 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.9.143.252:37215(127.9.143.252:37215)] connecting to peer
08:08:39.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:39.846091 9791 external_mini_cluster.cc:1428] Started /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu as pid 10907
08:08:39.858 [DEBUG - kudu-nio-4] (Connection.java:757) [peer master-127.9.143.252:37215(127.9.143.252:37215)] Successfully connected to peer
08:08:39.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:39.861634 11013 sys_catalog.cc:263] Verifying existing consensus state
08:08:39.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:39.869284 11013 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a: Bootstrap starting.
08:08:39.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:39.884521 11016 negotiation.cc:343] Unauthorized connection attempt: Server connection negotiation failed: server connection from 127.0.0.1:48882: token expired
08:08:39.889 [DEBUG - kudu-nio-4] (Negotiator.java:352) peer /127.9.143.252:37215 sent connection negotiation error: Not authorized: token expired
08:08:39.890 [DEBUG - kudu-nio-4] (Connection.java:239) [peer master-127.9.143.252:37215(127.9.143.252:37215)] handling channelInactive
08:08:39.890 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.9.143.252:37215(127.9.143.252:37215)] cleaning up while in state NEGOTIATION_FAILED due to: connection closed
08:08:39.892 [DEBUG - kudu-nio-4] (Connection.java:748) [peer master-127.9.143.252:37215(127.9.143.252:37215)] connecting to peer
08:08:39.893 [DEBUG - kudu-nio-4] (Connection.java:748) [peer master-127.9.143.254:42279(127.9.143.254:42279)] connecting to peer
08:08:39.895 [DEBUG - kudu-nio-4] (Connection.java:748) [peer master-127.9.143.253:32923(127.9.143.253:32923)] connecting to peer
08:08:39.903 [DEBUG - kudu-nio-6] (Connection.java:757) [peer master-127.9.143.254:42279(127.9.143.254:42279)] Successfully connected to peer
08:08:39.904 [DEBUG - kudu-nio-7] (Connection.java:757) [peer master-127.9.143.253:32923(127.9.143.253:32923)] Successfully connected to peer
08:08:39.916 [DEBUG - kudu-nio-5] (Connection.java:757) [peer master-127.9.143.252:37215(127.9.143.252:37215)] Successfully connected to peer
08:08:39.922 [DEBUG - kudu-nio-7] (SecurityContext.java:222) Refreshing Kerberos credentials...
08:08:39.922 [DEBUG - kudu-nio-7] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task_nU1WP/test-tmp/mini-kudu-cluster17378952590992577964/krb5kdc/krb5cc
08:08:39.924 [DEBUG - kudu-nio-7] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
08:08:39.924 [WARN - kudu-nio-7] (SecurityContext.java:231) Tried to refresh Kerberos credentials but was unable to re-login from ticket cache
08:08:39.924 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.9.143.253:32923(127.9.143.253:32923)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials
08:08:39.924 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.9.143.252:37215(127.9.143.252:37215)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials
08:08:39.925 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.9.143.253:32923: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials
08:08:39.925 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.9.143.252:37215: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials
08:08:39.925 [DEBUG - kudu-nio-7] (Connection.java:239) [peer master-127.9.143.253:32923(127.9.143.253:32923)] handling channelInactive
08:08:39.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:39.927069 11016 negotiation.cc:336] Failed RPC negotiation. Trace:
08:08:39.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 08:08:39.896149 (+ 0us) reactor.cc:678] Submitting negotiation task for server connection from 127.0.0.1:48894 (local address 127.9.143.252:37215)
08:08:39.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 08:08:39.896988 (+ 839us) server_negotiation.cc:207] Beginning negotiation
08:08:39.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 08:08:39.897017 (+ 29us) server_negotiation.cc:400] Waiting for connection header
08:08:39.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 08:08:39.917094 (+ 20077us) server_negotiation.cc:408] Connection header received
08:08:39.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 08:08:39.923193 (+ 6099us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
08:08:39.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 08:08:39.923227 (+ 34us) server_negotiation.cc:462] Received NEGOTIATE request from client
08:08:39.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 08:08:39.923547 (+ 320us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
08:08:39.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 08:08:39.923725 (+ 178us) server_negotiation.cc:227] Negotiated authn=SASL
08:08:39.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 08:08:39.925816 (+ 2091us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:48894: BlockingRecv error: recv got EOF from 127.0.0.1:48894 (error 108)
08:08:39.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":131}
08:08:39.928 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.9.143.254:42279(127.9.143.254:42279)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials
08:08:39.928 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.9.143.254:42279: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials
08:08:39.930 [DEBUG - kudu-nio-6] (Connection.java:239) [peer master-127.9.143.254:42279(127.9.143.254:42279)] handling channelInactive
08:08:39.930 [DEBUG - kudu-nio-5] (Connection.java:239) [peer master-127.9.143.252:37215(127.9.143.252:37215)] handling channelInactive
08:08:39.931 [DEBUG - main] (Connection.java:575) [peer master-127.9.143.252:37215(127.9.143.252:37215)] disconnecting while in state TERMINATED
08:08:39.931 [DEBUG - main] (Connection.java:575) [peer master-127.9.143.253:32923(127.9.143.253:32923)] disconnecting while in state TERMINATED
08:08:39.931 [DEBUG - main] (Connection.java:575) [peer master-127.9.143.254:42279(127.9.143.254:42279)] disconnecting while in state TERMINATED
08:08:39.931 [DEBUG - kudu-nio-5] (AsyncKuduClient.java:2704) Releasing all remaining resources
08:08:39.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:39.933542 11021 negotiation.cc:336] Failed RPC negotiation. Trace:
08:08:39.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 08:08:39.899077 (+ 0us) reactor.cc:678] Submitting negotiation task for server connection from 127.0.0.1:34152 (local address 127.9.143.254:42279)
08:08:39.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 08:08:39.925337 (+ 26260us) server_negotiation.cc:207] Beginning negotiation
08:08:39.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 08:08:39.925366 (+ 29us) server_negotiation.cc:400] Waiting for connection header
08:08:39.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 08:08:39.925420 (+ 54us) server_negotiation.cc:408] Connection header received
08:08:39.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 08:08:39.925576 (+ 156us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
08:08:39.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 08:08:39.925601 (+ 25us) server_negotiation.cc:462] Received NEGOTIATE request from client
08:08:39.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 08:08:39.927053 (+ 1452us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
08:08:39.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 08:08:39.927249 (+ 196us) server_negotiation.cc:227] Negotiated authn=SASL
08:08:39.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 08:08:39.932165 (+ 4916us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:34152: BlockingRecv error: recv got EOF from 127.0.0.1:34152 (error 108)
08:08:39.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":25662,"thread_start_us":25898,"threads_started":1}
08:08:39.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:39.934257 9791 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu with pid 10814
08:08:39.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 08:08:39.943383 11020 negotiation.cc:336] Failed RPC negotiation. Trace:
08:08:39.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 08:08:39.898519 (+ 0us) reactor.cc:678] Submitting negotiation task for server connection from 127.0.0.1:49308 (local address 127.9.143.253:32923)
08:08:39.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 08:08:39.914241 (+ 15722us) server_negotiation.cc:207] Beginning negotiation
08:08:39.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 08:08:39.914284 (+ 43us) server_negotiation.cc:400] Waiting for connection header
08:08:39.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 08:08:39.914393 (+ 109us) server_negotiation.cc:408] Connection header received
08:08:39.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 08:08:39.920487 (+ 6094us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
08:08:39.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 08:08:39.920554 (+ 67us) server_negotiation.cc:462] Received NEGOTIATE request from client
08:08:39.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 08:08:39.921069 (+ 515us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
08:08:39.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 08:08:39.927434 (+ 6365us) server_negotiation.cc:227] Negotiated authn=SASL
08:08:39.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 08:08:39.928648 (+ 1214us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:49308: BlockingRecv error: recv got EOF from 127.0.0.1:49308 (error 108)
08:08:39.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":14896,"thread_start_us":15743,"threads_started":1}
08:08:39.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:39.950580 11013 log.cc:826] T 00000000000000000000000000000000 P 1d36c7228de444508830a92c33ddaa4a: Log is configured to *not* fsync() on all Append() calls
08:08:39.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:39.973152 9791 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu with pid 10907
08:08:39.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:39.987697 9791 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu with pid 10742
08:08:41.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-03T08:08:41Z chronyd exiting
08:08:42.140 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
08:08:42.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:42.143352 9065 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu with pid 9346
08:08:42.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:42.166769 9065 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu with pid 9507
08:08:42.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:42.188841 9065 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu with pid 9650
08:08:42.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:42.210389 9065 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu with pid 9077
08:08:42.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:42.238535 9065 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu with pid 9147
08:08:42.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 08:08:42.264030 9065 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task_nU1WP/build/tsan/bin/kudu with pid 9222
08:08:43.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-03T08:08:43Z chronyd exiting
08:08:44.361 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testDoNotSwitchPrincipalsInExistingClient: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot re-acquire authentication token after 5 attempts (Couldn't find a valid master in (127.9.143.252:37215,127.9.143.254:42279,127.9.143.253:32923). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials])
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386)
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40)
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378)
at org.apache.kudu.client.TestSecurity.testDoNotSwitchPrincipalsInExistingClient(TestSecurity.java:366)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:158)
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:141)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot re-acquire authentication token after 5 attempts (Couldn't find a valid master in (127.9.143.252:37215,127.9.143.254:42279,127.9.143.253:32923). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials])
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386)
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40)
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378)
at org.apache.kudu.client.TestSecurity.testDoNotSwitchPrincipalsInExistingClient(TestSecurity.java:366)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:158)
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:141)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
08:08:45.054 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testDoNotSwitchPrincipalsInExistingClient: giving up after 1 attempts
08:08:45.055 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testDoNotSwitchPrincipalsInExistingClient: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot re-acquire authentication token after 5 attempts (Couldn't find a valid master in (127.9.143.252:37215,127.9.143.254:42279,127.9.143.253:32923). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials])
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386)
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40)
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378)
at org.apache.kudu.client.TestSecurity.testDoNotSwitchPrincipalsInExistingClient(TestSecurity.java:366)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:158)
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:141)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot re-acquire authentication token after 5 attempts (Couldn't find a valid master in (127.9.143.252:37215,127.9.143.254:42279,127.9.143.253:32923). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials])
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386)
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40)
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378)
at org.apache.kudu.client.TestSecurity.testDoNotSwitchPrincipalsInExistingClient(TestSecurity.java:366)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:158)
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:141)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more