21:12:48.465 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
21:12:48.465 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskpJdSDR/test-tmp
21:12:48.466 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
21:12:48.466 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskpJdSDR/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
21:12:48.467 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
21:12:48.793 [DEBUG - main] (MiniKuduCluster.java:195) Response:
21:12:48.794 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
21:12:48.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:48.796888 5048 mini_oidc.cc:150] Starting JWKS server
21:12:48.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:48.799963 5048 webserver.cc:492] Webserver started at http://127.0.0.1:39665/ using document root <none> and password file <none>
21:12:48.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:48.800372 5048 mini_oidc.cc:186] Starting OIDC Discovery server
21:12:48.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:48.802080 5048 webserver.cc:492] Webserver started at http://127.0.0.1:36571/ using document root <none> and password file <none>
21:12:48.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-11T21:12:48Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
21:12:48.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-11T21:12:48Z Disabled control of system clock
21:12:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:48.860625 5048 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu
21:12:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu
21:12:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-0/wal
21:12:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-0/data
21:12:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:12:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:12:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:12:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:12:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:12:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:12:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-0/logs
21:12:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-0/data/info.pb
21:12:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:12:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:12:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:12:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:12:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:12:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:12:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:12:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:12:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:12:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:12:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:12:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:12:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.238.62:45103
21:12:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.238.62
21:12:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:12:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.238.20:36399
21:12:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:12:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:12:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:12:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:12:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.238.62:45103,127.4.238.61:41491,127.4.238.60:33899
21:12:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:12:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39665/jwks.json with env {}
21:12:49.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:49.150513 5060 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:12:49.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:49.151065 5060 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:12:49.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:49.151543 5060 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:12:49.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:49.183483 5060 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:12:49.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:49.183794 5060 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:12:49.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:49.184015 5060 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:12:49.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:49.184209 5060 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:12:49.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:49.184401 5060 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:12:49.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:49.184544 5060 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39665/jwks.json
21:12:49.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:49.223992 5060 master_runner.cc:386] Master server non-default flags:
21:12:49.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:12:49.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.238.20:36399
21:12:49.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:12:49.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:12:49.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-0/data
21:12:49.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-0/wal
21:12:49.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:12:49.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.238.62:45103,127.4.238.61:41491,127.4.238.60:33899
21:12:49.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:12:49.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:12:49.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:12:49.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.238.62:45103
21:12:49.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:12:49.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:12:49.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:12:49.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39665/jwks.json
21:12:49.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:12:49.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:12:49.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-0/data/info.pb
21:12:49.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.238.62
21:12:49.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:12:49.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:12:49.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:12:49.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:12:49.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:12:49.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:12:49.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-0/logs
21:12:49.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:12:49.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:12:49.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:12:49.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:12:49.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:12:49.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
21:12:49.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:12:49.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Dec 2025 20:43:24 UTC on 5fd53c4cbb9d
21:12:49.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9469
21:12:49.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:12:49.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:49.225257 5060 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:12:49.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:49.226951 5060 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:12:49.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:49.238577 5066 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:12:50.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:49.238580 5067 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:12:50.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:50.344396 5060 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.106s user 0.002s sys 0.003s
21:12:50.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:50.344506 5068 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1101 milliseconds
21:12:50.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:50.344970 5060 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.107s user 0.002s sys 0.004s
21:12:50.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:50.346149 5069 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:12:50.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:50.346227 5060 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:12:50.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:50.347483 5060 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:12:50.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:50.350209 5060 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:12:50.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:50.351603 5060 hybrid_clock.cc:648] HybridClock initialized: now 1765487570351543 us; error 65 us; skew 500 ppm
21:12:50.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:50.352378 5060 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:12:50.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:50.359544 5060 webserver.cc:492] Webserver started at http://127.4.238.62:43101/ using document root <none> and password file <none>
21:12:50.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:50.360404 5060 fs_manager.cc:362] Metadata directory not provided
21:12:50.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:50.360582 5060 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:12:50.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:50.360961 5060 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:12:50.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:50.365160 5060 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-0/data/instance:
21:12:50.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "81807fdfb68d46dc94b3a84d3f539416"
21:12:50.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-11 21:12:50 on dist-test-slave-2lgj"
21:12:50.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:50.366246 5060 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-0/wal/instance:
21:12:50.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "81807fdfb68d46dc94b3a84d3f539416"
21:12:50.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-11 21:12:50 on dist-test-slave-2lgj"
21:12:50.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:50.373777 5060 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.000s
21:12:50.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:50.379601 5076 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:12:50.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:50.380820 5060 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
21:12:50.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:50.381160 5060 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-0/data,/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-0/wal
21:12:50.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "81807fdfb68d46dc94b3a84d3f539416"
21:12:50.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-11 21:12:50 on dist-test-slave-2lgj"
21:12:50.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:50.381480 5060 fs_report.cc:389] FS layout report
21:12:50.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:12:50.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-0/wal
21:12:50.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-0/wal
21:12:50.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-0/data/data
21:12:50.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:12:50.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:12:50.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:12:50.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:12:50.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:12:50.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:12:50.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:12:50.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:12:50.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:12:50.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:12:50.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:12:50.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:50.443377 5060 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:12:50.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:50.444918 5060 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:12:50.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:50.445363 5060 kserver.cc:163] Server-wide thread pool size limit: 3276
21:12:50.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:50.511581 5060 rpc_server.cc:307] RPC server started. Bound to: 127.4.238.62:45103
21:12:50.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:50.511636 5127 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.238.62:45103 every 8 connection(s)
21:12:50.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:50.514266 5060 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-0/data/info.pb
21:12:50.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:50.516762 5048 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu as pid 5060
21:12:50.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:50.517323 5048 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-0/wal/instance
21:12:50.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:50.520263 5128 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:12:50.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:50.520608 5048 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu
21:12:50.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu
21:12:50.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-1/wal
21:12:50.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-1/data
21:12:50.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:12:50.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:12:50.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:12:50.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:12:50.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:12:50.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:12:50.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-1/logs
21:12:50.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-1/data/info.pb
21:12:50.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:12:50.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:12:50.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:12:50.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:12:50.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:12:50.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:12:50.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:12:50.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:12:50.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:12:50.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:12:50.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:12:50.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:12:50.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.238.61:41491
21:12:50.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.238.61
21:12:50.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:12:50.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.238.20:36399
21:12:50.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:12:50.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:12:50.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:12:50.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:12:50.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.238.62:45103,127.4.238.61:41491,127.4.238.60:33899
21:12:50.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:12:50.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39665/jwks.json with env {}
21:12:50.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:50.533210 5128 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.238.62" port: 45103 } has no permanent_uuid. Determining permanent_uuid...
21:12:50.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:50.560842 5128 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.238.61" port: 41491 } has no permanent_uuid. Determining permanent_uuid...
21:12:50.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:50.563401 5080 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.238.61:41491: connect: Connection refused (error 111)
21:12:50.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:50.567492 5128 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.238.61:41491: Network error: Client connection negotiation failed: client connection to 127.4.238.61:41491: connect: Connection refused (error 111)
21:12:50.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:50.616940 5128 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.238.61" port: 41491 } attempt: 1
21:12:50.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:50.620888 5128 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.238.61:41491: Network error: Client connection negotiation failed: client connection to 127.4.238.61:41491: connect: Connection refused (error 111)
21:12:50.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:50.689270 5128 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.238.61" port: 41491 } attempt: 2
21:12:50.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:50.692896 5128 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.238.61:41491: Network error: Client connection negotiation failed: client connection to 127.4.238.61:41491: connect: Connection refused (error 111)
21:12:50.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:50.784281 5128 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.238.61" port: 41491 } attempt: 3
21:12:50.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:50.788636 5128 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.238.61:41491: Network error: Client connection negotiation failed: client connection to 127.4.238.61:41491: connect: Connection refused (error 111)
21:12:50.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:50.819550 5130 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:12:50.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:50.820077 5130 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:12:50.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:50.820577 5130 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:12:50.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:50.851784 5130 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:12:50.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:50.852066 5130 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:12:50.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:50.852264 5130 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:12:50.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:50.852452 5130 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:12:50.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:50.852639 5130 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:12:50.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:50.852783 5130 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39665/jwks.json
21:12:50.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:50.887805 5130 master_runner.cc:386] Master server non-default flags:
21:12:50.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:12:50.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.238.20:36399
21:12:50.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:12:50.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:12:50.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-1/data
21:12:50.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-1/wal
21:12:50.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:12:50.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.238.62:45103,127.4.238.61:41491,127.4.238.60:33899
21:12:50.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:12:50.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:12:50.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:12:50.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.238.61:41491
21:12:50.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:12:50.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:12:50.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:12:50.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39665/jwks.json
21:12:50.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:12:50.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:12:50.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-1/data/info.pb
21:12:50.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.238.61
21:12:50.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:12:50.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:12:50.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:12:50.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:12:50.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:12:50.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:12:50.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-1/logs
21:12:50.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:12:50.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:12:50.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:12:50.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:12:50.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:12:50.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
21:12:50.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:12:50.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Dec 2025 20:43:24 UTC on 5fd53c4cbb9d
21:12:50.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9469
21:12:50.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:12:50.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:50.889135 5130 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:12:50.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:50.890949 5130 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:12:50.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:50.900727 5139 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:12:50.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:50.931998 5128 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.238.61" port: 41491 } attempt: 4
21:12:50.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:50.935591 5128 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.238.61:41491: Network error: Client connection negotiation failed: client connection to 127.4.238.61:41491: connect: Connection refused (error 111)
21:12:51.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:51.235172 5128 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.238.61" port: 41491 } attempt: 5
21:12:51.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:51.239171 5128 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.238.61:41491: Network error: Client connection negotiation failed: client connection to 127.4.238.61:41491: connect: Connection refused (error 111)
21:12:51.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:51.786641 5128 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.238.61" port: 41491 } attempt: 6
21:12:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:51.798434 5128 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.238.61:41491: Network error: Client connection negotiation failed: client connection to 127.4.238.61:41491: connect: Connection refused (error 111)
21:12:52.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:50.901949 5140 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:12:52.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:50.905073 5142 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:12:52.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:52.001652 5141 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
21:12:52.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:52.001816 5130 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:12:52.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:52.005226 5130 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:12:52.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:52.008162 5130 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:12:52.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:52.009565 5130 hybrid_clock.cc:648] HybridClock initialized: now 1765487572009529 us; error 56 us; skew 500 ppm
21:12:52.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:52.010406 5130 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:12:52.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:52.016535 5130 webserver.cc:492] Webserver started at http://127.4.238.61:33921/ using document root <none> and password file <none>
21:12:52.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:52.017416 5130 fs_manager.cc:362] Metadata directory not provided
21:12:52.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:52.017606 5130 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:12:52.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:52.018049 5130 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:12:52.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:52.022286 5130 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-1/data/instance:
21:12:52.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8c99478dab94407b8f06ab5b292ab0d4"
21:12:52.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-11 21:12:52 on dist-test-slave-2lgj"
21:12:52.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:52.023334 5130 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-1/wal/instance:
21:12:52.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8c99478dab94407b8f06ab5b292ab0d4"
21:12:52.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-11 21:12:52 on dist-test-slave-2lgj"
21:12:52.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:52.030161 5130 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.004s sys 0.004s
21:12:52.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:52.035786 5151 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:12:52.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:52.037318 5130 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
21:12:52.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:52.037744 5130 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-1/data,/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-1/wal
21:12:52.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8c99478dab94407b8f06ab5b292ab0d4"
21:12:52.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-11 21:12:52 on dist-test-slave-2lgj"
21:12:52.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:52.038208 5130 fs_report.cc:389] FS layout report
21:12:52.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:12:52.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-1/wal
21:12:52.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-1/wal
21:12:52.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-1/data/data
21:12:52.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:12:52.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:12:52.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:12:52.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:12:52.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:12:52.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:12:52.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:12:52.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:12:52.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:12:52.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:12:52.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:12:52.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:52.114152 5130 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:12:52.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:52.115700 5130 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:12:52.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:52.116264 5130 kserver.cc:163] Server-wide thread pool size limit: 3276
21:12:52.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:52.183558 5130 rpc_server.cc:307] RPC server started. Bound to: 127.4.238.61:41491
21:12:52.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:52.183652 5202 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.238.61:41491 every 8 connection(s)
21:12:52.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:52.186332 5130 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-1/data/info.pb
21:12:52.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:52.192117 5203 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:12:52.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:52.196333 5048 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu as pid 5130
21:12:52.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:52.196686 5048 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-1/wal/instance
21:12:52.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:52.198676 5048 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu
21:12:52.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu
21:12:52.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-2/wal
21:12:52.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-2/data
21:12:52.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:12:52.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:12:52.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:12:52.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:12:52.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:12:52.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:12:52.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-2/logs
21:12:52.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-2/data/info.pb
21:12:52.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:12:52.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:12:52.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:12:52.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:12:52.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:12:52.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:12:52.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:12:52.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:12:52.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:12:52.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:12:52.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:12:52.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:12:52.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.238.60:33899
21:12:52.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.238.60
21:12:52.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:12:52.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.238.20:36399
21:12:52.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:12:52.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:12:52.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:12:52.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:12:52.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.238.62:45103,127.4.238.61:41491,127.4.238.60:33899
21:12:52.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:12:52.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39665/jwks.json with env {}
21:12:52.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:52.203480 5203 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.238.62" port: 45103 } has no permanent_uuid. Determining permanent_uuid...
21:12:52.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:52.225485 5203 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.238.61" port: 41491 } has no permanent_uuid. Determining permanent_uuid...
21:12:52.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:52.239162 5203 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.238.60" port: 33899 } has no permanent_uuid. Determining permanent_uuid...
21:12:52.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:52.241417 5155 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.238.60:33899: connect: Connection refused (error 111)
21:12:52.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:52.245374 5203 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.238.60:33899: Network error: Client connection negotiation failed: client connection to 127.4.238.60:33899: connect: Connection refused (error 111)
21:12:52.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:52.294831 5203 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.238.60" port: 33899 } attempt: 1
21:12:52.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:52.298629 5203 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.238.60:33899: Network error: Client connection negotiation failed: client connection to 127.4.238.60:33899: connect: Connection refused (error 111)
21:12:52.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:52.366999 5203 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.238.60" port: 33899 } attempt: 2
21:12:52.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:52.371236 5203 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.238.60:33899: Network error: Client connection negotiation failed: client connection to 127.4.238.60:33899: connect: Connection refused (error 111)
21:12:52.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:52.462643 5203 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.238.60" port: 33899 } attempt: 3
21:12:52.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:52.466634 5203 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.238.60:33899: Network error: Client connection negotiation failed: client connection to 127.4.238.60:33899: connect: Connection refused (error 111)
21:12:52.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:52.502635 5205 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:12:52.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:52.503263 5205 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:12:52.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:52.503841 5205 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:12:52.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:52.535741 5205 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:12:52.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:52.536083 5205 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:12:52.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:52.536371 5205 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:12:52.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:52.536638 5205 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:12:52.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:52.536911 5205 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:12:52.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:52.537113 5205 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39665/jwks.json
21:12:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:52.572136 5205 master_runner.cc:386] Master server non-default flags:
21:12:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:12:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.238.20:36399
21:12:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:12:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:12:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-2/data
21:12:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-2/wal
21:12:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:12:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.238.62:45103,127.4.238.61:41491,127.4.238.60:33899
21:12:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:12:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:12:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:12:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.238.60:33899
21:12:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:12:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:12:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:12:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39665/jwks.json
21:12:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:12:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:12:52.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-2/data/info.pb
21:12:52.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.238.60
21:12:52.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:12:52.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:12:52.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:12:52.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:12:52.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:12:52.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:12:52.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-2/logs
21:12:52.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:12:52.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:12:52.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:12:52.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:12:52.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:12:52.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
21:12:52.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:12:52.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Dec 2025 20:43:24 UTC on 5fd53c4cbb9d
21:12:52.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9469
21:12:52.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:12:52.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:52.573712 5205 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:12:52.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:52.575359 5205 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:12:52.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:52.586144 5215 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:12:52.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:52.610034 5203 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.238.60" port: 33899 } attempt: 4
21:12:52.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:52.616288 5203 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.238.60:33899: Network error: Client connection negotiation failed: client connection to 127.4.238.60:33899: connect: Connection refused (error 111)
21:12:52.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:52.858906 5128 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.238.61" port: 41491 } attempt: 7
21:12:52.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:52.901130 5128 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.238.60" port: 33899 } has no permanent_uuid. Determining permanent_uuid...
21:12:52.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:52.915735 5203 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.238.60" port: 33899 } attempt: 5
21:12:52.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:52.917804 5128 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.238.60:33899: Network error: Client connection negotiation failed: client connection to 127.4.238.60:33899: connect: Connection refused (error 111)
21:12:52.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:52.920883 5203 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.238.60:33899: Network error: Client connection negotiation failed: client connection to 127.4.238.60:33899: connect: Connection refused (error 111)
21:12:52.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:52.976280 5128 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.238.60" port: 33899 } attempt: 1
21:12:52.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:52.982213 5128 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.238.60:33899: Network error: Client connection negotiation failed: client connection to 127.4.238.60:33899: connect: Connection refused (error 111)
21:12:53.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:53.063656 5128 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.238.60" port: 33899 } attempt: 2
21:12:53.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:53.069509 5128 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.238.60:33899: Network error: Client connection negotiation failed: client connection to 127.4.238.60:33899: connect: Connection refused (error 111)
21:12:53.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:53.155187 5128 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.238.60" port: 33899 } attempt: 3
21:12:53.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:53.160900 5128 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.238.60:33899: Network error: Client connection negotiation failed: client connection to 127.4.238.60:33899: connect: Connection refused (error 111)
21:12:53.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:53.301520 5128 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.238.60" port: 33899 } attempt: 4
21:12:53.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:53.307550 5128 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.238.60:33899: Network error: Client connection negotiation failed: client connection to 127.4.238.60:33899: connect: Connection refused (error 111)
21:12:53.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:53.468327 5203 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.238.60" port: 33899 } attempt: 6
21:12:53.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:53.494124 5203 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.238.60:33899: Network error: Client connection negotiation failed: client connection to 127.4.238.60:33899: connect: Connection refused (error 111)
21:12:53.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:53.591053 5128 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.238.60" port: 33899 } attempt: 5
21:12:53.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:53.597611 5128 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.238.60:33899: Network error: Client connection negotiation failed: client connection to 127.4.238.60:33899: connect: Connection refused (error 111)
21:12:53.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:52.586781 5216 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:12:53.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:53.906050 5217 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1318 milliseconds
21:12:53.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:53.907250 5218 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:12:53.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:53.910209 5205 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.324s user 0.516s sys 0.803s
21:12:53.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:53.910543 5205 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.325s user 0.516s sys 0.803s
21:12:53.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:53.910830 5205 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:12:53.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:53.912259 5205 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:12:53.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:53.915491 5205 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:12:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:53.916914 5205 hybrid_clock.cc:648] HybridClock initialized: now 1765487573916827 us; error 94 us; skew 500 ppm
21:12:53.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:53.918022 5205 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:12:53.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:53.926302 5205 webserver.cc:492] Webserver started at http://127.4.238.60:45651/ using document root <none> and password file <none>
21:12:53.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:53.927608 5205 fs_manager.cc:362] Metadata directory not provided
21:12:53.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:53.927891 5205 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:12:53.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:53.928450 5205 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:12:53.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:53.934728 5205 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-2/data/instance:
21:12:53.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bf9e588597644890990196db87d21a52"
21:12:53.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-11 21:12:53 on dist-test-slave-2lgj"
21:12:53.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:53.936198 5205 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-2/wal/instance:
21:12:53.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bf9e588597644890990196db87d21a52"
21:12:53.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-11 21:12:53 on dist-test-slave-2lgj"
21:12:53.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:53.945425 5205 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.009s sys 0.001s
21:12:53.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:53.952533 5231 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:12:53.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:53.953662 5205 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.003s
21:12:53.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:53.953974 5205 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-2/data,/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-2/wal
21:12:53.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bf9e588597644890990196db87d21a52"
21:12:53.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-11 21:12:53 on dist-test-slave-2lgj"
21:12:53.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:53.954273 5205 fs_report.cc:389] FS layout report
21:12:53.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:12:53.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-2/wal
21:12:53.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-2/wal
21:12:53.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-2/data/data
21:12:53.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:12:53.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:12:53.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:12:53.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:12:53.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:12:53.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:12:53.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:12:53.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:12:53.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:12:53.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:12:53.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:12:54.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.023324 5205 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:12:54.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.024705 5205 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:12:54.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.025099 5205 kserver.cc:163] Server-wide thread pool size limit: 3276
21:12:54.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.091840 5205 rpc_server.cc:307] RPC server started. Bound to: 127.4.238.60:33899
21:12:54.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.091919 5282 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.238.60:33899 every 8 connection(s)
21:12:54.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.094398 5205 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-2/data/info.pb
21:12:54.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.099026 5048 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu as pid 5205
21:12:54.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.099399 5048 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/master-2/wal/instance
21:12:54.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.100253 5283 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:12:54.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.114086 5283 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.238.62" port: 45103 } has no permanent_uuid. Determining permanent_uuid...
21:12:54.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.135777 5283 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.238.61" port: 41491 } has no permanent_uuid. Determining permanent_uuid...
21:12:54.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.145375 5283 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.238.60" port: 33899 } has no permanent_uuid. Determining permanent_uuid...
21:12:54.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.150135 5128 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.238.60" port: 33899 } attempt: 6
21:12:54.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.161875 5283 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bf9e588597644890990196db87d21a52: Bootstrap starting.
21:12:54.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.168255 5283 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P bf9e588597644890990196db87d21a52: Neither blocks nor log segments found. Creating new log.
21:12:54.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.170374 5128 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 81807fdfb68d46dc94b3a84d3f539416: Bootstrap starting.
21:12:54.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.170575 5283 log.cc:826] T 00000000000000000000000000000000 P bf9e588597644890990196db87d21a52: Log is configured to *not* fsync() on all Append() calls
21:12:54.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.175561 5283 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bf9e588597644890990196db87d21a52: No bootstrap required, opened a new log
21:12:54.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.176597 5128 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 81807fdfb68d46dc94b3a84d3f539416: Neither blocks nor log segments found. Creating new log.
21:12:54.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.178862 5128 log.cc:826] T 00000000000000000000000000000000 P 81807fdfb68d46dc94b3a84d3f539416: Log is configured to *not* fsync() on all Append() calls
21:12:54.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.184154 5128 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 81807fdfb68d46dc94b3a84d3f539416: No bootstrap required, opened a new log
21:12:54.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.201386 5283 raft_consensus.cc:359] T 00000000000000000000000000000000 P bf9e588597644890990196db87d21a52 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81807fdfb68d46dc94b3a84d3f539416" member_type: VOTER last_known_addr { host: "127.4.238.62" port: 45103 } } peers { permanent_uuid: "8c99478dab94407b8f06ab5b292ab0d4" member_type: VOTER last_known_addr { host: "127.4.238.61" port: 41491 } } peers { permanent_uuid: "bf9e588597644890990196db87d21a52" member_type: VOTER last_known_addr { host: "127.4.238.60" port: 33899 } }
21:12:54.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.202306 5283 raft_consensus.cc:385] T 00000000000000000000000000000000 P bf9e588597644890990196db87d21a52 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:12:54.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.202593 5283 raft_consensus.cc:740] T 00000000000000000000000000000000 P bf9e588597644890990196db87d21a52 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: bf9e588597644890990196db87d21a52, State: Initialized, Role: FOLLOWER
21:12:54.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.203548 5283 consensus_queue.cc:260] T 00000000000000000000000000000000 P bf9e588597644890990196db87d21a52 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81807fdfb68d46dc94b3a84d3f539416" member_type: VOTER last_known_addr { host: "127.4.238.62" port: 45103 } } peers { permanent_uuid: "8c99478dab94407b8f06ab5b292ab0d4" member_type: VOTER last_known_addr { host: "127.4.238.61" port: 41491 } } peers { permanent_uuid: "bf9e588597644890990196db87d21a52" member_type: VOTER last_known_addr { host: "127.4.238.60" port: 33899 } }
21:12:54.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.206094 5128 raft_consensus.cc:359] T 00000000000000000000000000000000 P 81807fdfb68d46dc94b3a84d3f539416 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81807fdfb68d46dc94b3a84d3f539416" member_type: VOTER last_known_addr { host: "127.4.238.62" port: 45103 } } peers { permanent_uuid: "8c99478dab94407b8f06ab5b292ab0d4" member_type: VOTER last_known_addr { host: "127.4.238.61" port: 41491 } } peers { permanent_uuid: "bf9e588597644890990196db87d21a52" member_type: VOTER last_known_addr { host: "127.4.238.60" port: 33899 } }
21:12:54.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.207006 5128 raft_consensus.cc:385] T 00000000000000000000000000000000 P 81807fdfb68d46dc94b3a84d3f539416 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:12:54.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.207401 5128 raft_consensus.cc:740] T 00000000000000000000000000000000 P 81807fdfb68d46dc94b3a84d3f539416 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 81807fdfb68d46dc94b3a84d3f539416, State: Initialized, Role: FOLLOWER
21:12:54.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.208341 5128 consensus_queue.cc:260] T 00000000000000000000000000000000 P 81807fdfb68d46dc94b3a84d3f539416 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81807fdfb68d46dc94b3a84d3f539416" member_type: VOTER last_known_addr { host: "127.4.238.62" port: 45103 } } peers { permanent_uuid: "8c99478dab94407b8f06ab5b292ab0d4" member_type: VOTER last_known_addr { host: "127.4.238.61" port: 41491 } } peers { permanent_uuid: "bf9e588597644890990196db87d21a52" member_type: VOTER last_known_addr { host: "127.4.238.60" port: 33899 } }
21:12:54.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.210696 5295 sys_catalog.cc:455] T 00000000000000000000000000000000 P bf9e588597644890990196db87d21a52 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81807fdfb68d46dc94b3a84d3f539416" member_type: VOTER last_known_addr { host: "127.4.238.62" port: 45103 } } peers { permanent_uuid: "8c99478dab94407b8f06ab5b292ab0d4" member_type: VOTER last_known_addr { host: "127.4.238.61" port: 41491 } } peers { permanent_uuid: "bf9e588597644890990196db87d21a52" member_type: VOTER last_known_addr { host: "127.4.238.60" port: 33899 } } }
21:12:54.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.211440 5295 sys_catalog.cc:458] T 00000000000000000000000000000000 P bf9e588597644890990196db87d21a52 [sys.catalog]: This master's current role is: FOLLOWER
21:12:54.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.214316 5283 sys_catalog.cc:565] T 00000000000000000000000000000000 P bf9e588597644890990196db87d21a52 [sys.catalog]: configured and running, proceeding with master startup.
21:12:54.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.215133 5296 sys_catalog.cc:455] T 00000000000000000000000000000000 P 81807fdfb68d46dc94b3a84d3f539416 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81807fdfb68d46dc94b3a84d3f539416" member_type: VOTER last_known_addr { host: "127.4.238.62" port: 45103 } } peers { permanent_uuid: "8c99478dab94407b8f06ab5b292ab0d4" member_type: VOTER last_known_addr { host: "127.4.238.61" port: 41491 } } peers { permanent_uuid: "bf9e588597644890990196db87d21a52" member_type: VOTER last_known_addr { host: "127.4.238.60" port: 33899 } } }
21:12:54.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.215878 5296 sys_catalog.cc:458] T 00000000000000000000000000000000 P 81807fdfb68d46dc94b3a84d3f539416 [sys.catalog]: This master's current role is: FOLLOWER
21:12:54.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.217361 5128 sys_catalog.cc:565] T 00000000000000000000000000000000 P 81807fdfb68d46dc94b3a84d3f539416 [sys.catalog]: configured and running, proceeding with master startup.
21:12:54.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:54.248979 5316 catalog_manager.cc:1568] T 00000000000000000000000000000000 P bf9e588597644890990196db87d21a52: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:12:54.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:54.249307 5316 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:12:54.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:54.256860 5318 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 81807fdfb68d46dc94b3a84d3f539416: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:12:54.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:54.257206 5318 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:12:54.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:54.280010 5237 tablet.cc:2378] T 00000000000000000000000000000000 P bf9e588597644890990196db87d21a52: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:12:54.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.243382 5205 master_runner.cc:186] Error getting master registration for 127.4.238.62:45103: OK, instance_id { permanent_uuid: "81807fdfb68d46dc94b3a84d3f539416" instance_seqno: 1765487570493336 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:12:54.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.286121 5205 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:12:54.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.254109 5060 master_runner.cc:186] Error getting master registration for 127.4.238.61:41491: OK, instance_id { permanent_uuid: "8c99478dab94407b8f06ab5b292ab0d4" instance_seqno: 1765487572165524 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:12:54.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.290299 5060 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:12:54.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:54.453994 5082 tablet.cc:2378] T 00000000000000000000000000000000 P 81807fdfb68d46dc94b3a84d3f539416: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:12:54.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.498991 5295 raft_consensus.cc:493] T 00000000000000000000000000000000 P bf9e588597644890990196db87d21a52 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:12:54.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.499547 5295 raft_consensus.cc:515] T 00000000000000000000000000000000 P bf9e588597644890990196db87d21a52 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81807fdfb68d46dc94b3a84d3f539416" member_type: VOTER last_known_addr { host: "127.4.238.62" port: 45103 } } peers { permanent_uuid: "8c99478dab94407b8f06ab5b292ab0d4" member_type: VOTER last_known_addr { host: "127.4.238.61" port: 41491 } } peers { permanent_uuid: "bf9e588597644890990196db87d21a52" member_type: VOTER last_known_addr { host: "127.4.238.60" port: 33899 } }
21:12:54.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.501861 5295 leader_election.cc:290] T 00000000000000000000000000000000 P bf9e588597644890990196db87d21a52 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 81807fdfb68d46dc94b3a84d3f539416 (127.4.238.62:45103), 8c99478dab94407b8f06ab5b292ab0d4 (127.4.238.61:41491)
21:12:54.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.502964 5103 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "bf9e588597644890990196db87d21a52" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "81807fdfb68d46dc94b3a84d3f539416" is_pre_election: true
21:12:54.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.503825 5103 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 81807fdfb68d46dc94b3a84d3f539416 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate bf9e588597644890990196db87d21a52 in term 0.
21:12:54.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.505467 5233 leader_election.cc:304] T 00000000000000000000000000000000 P bf9e588597644890990196db87d21a52 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 81807fdfb68d46dc94b3a84d3f539416, bf9e588597644890990196db87d21a52; no voters:
21:12:54.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.506771 5295 raft_consensus.cc:2804] T 00000000000000000000000000000000 P bf9e588597644890990196db87d21a52 [term 0 FOLLOWER]: Leader pre-election won for term 1
21:12:54.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.507514 5295 raft_consensus.cc:493] T 00000000000000000000000000000000 P bf9e588597644890990196db87d21a52 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:12:54.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.508241 5295 raft_consensus.cc:3060] T 00000000000000000000000000000000 P bf9e588597644890990196db87d21a52 [term 0 FOLLOWER]: Advancing to term 1
21:12:54.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.515245 5295 raft_consensus.cc:515] T 00000000000000000000000000000000 P bf9e588597644890990196db87d21a52 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81807fdfb68d46dc94b3a84d3f539416" member_type: VOTER last_known_addr { host: "127.4.238.62" port: 45103 } } peers { permanent_uuid: "8c99478dab94407b8f06ab5b292ab0d4" member_type: VOTER last_known_addr { host: "127.4.238.61" port: 41491 } } peers { permanent_uuid: "bf9e588597644890990196db87d21a52" member_type: VOTER last_known_addr { host: "127.4.238.60" port: 33899 } }
21:12:54.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.519816 5103 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "bf9e588597644890990196db87d21a52" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "81807fdfb68d46dc94b3a84d3f539416"
21:12:54.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.520382 5103 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 81807fdfb68d46dc94b3a84d3f539416 [term 0 FOLLOWER]: Advancing to term 1
21:12:54.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.527959 5103 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 81807fdfb68d46dc94b3a84d3f539416 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate bf9e588597644890990196db87d21a52 in term 1.
21:12:54.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.529330 5233 leader_election.cc:304] T 00000000000000000000000000000000 P bf9e588597644890990196db87d21a52 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 81807fdfb68d46dc94b3a84d3f539416, bf9e588597644890990196db87d21a52; no voters:
21:12:54.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.536681 5321 raft_consensus.cc:2804] T 00000000000000000000000000000000 P bf9e588597644890990196db87d21a52 [term 1 FOLLOWER]: Leader election won for term 1
21:12:54.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.539057 5295 leader_election.cc:290] T 00000000000000000000000000000000 P bf9e588597644890990196db87d21a52 [CANDIDATE]: Term 1 election: Requested vote from peers 81807fdfb68d46dc94b3a84d3f539416 (127.4.238.62:45103), 8c99478dab94407b8f06ab5b292ab0d4 (127.4.238.61:41491)
21:12:54.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.543450 5321 raft_consensus.cc:697] T 00000000000000000000000000000000 P bf9e588597644890990196db87d21a52 [term 1 LEADER]: Becoming Leader. State: Replica: bf9e588597644890990196db87d21a52, State: Running, Role: LEADER
21:12:54.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.544704 5321 consensus_queue.cc:237] T 00000000000000000000000000000000 P bf9e588597644890990196db87d21a52 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81807fdfb68d46dc94b3a84d3f539416" member_type: VOTER last_known_addr { host: "127.4.238.62" port: 45103 } } peers { permanent_uuid: "8c99478dab94407b8f06ab5b292ab0d4" member_type: VOTER last_known_addr { host: "127.4.238.61" port: 41491 } } peers { permanent_uuid: "bf9e588597644890990196db87d21a52" member_type: VOTER last_known_addr { host: "127.4.238.60" port: 33899 } }
21:12:54.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.554605 5203 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.238.60" port: 33899 } attempt: 7
21:12:54.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.574388 5295 sys_catalog.cc:455] T 00000000000000000000000000000000 P bf9e588597644890990196db87d21a52 [sys.catalog]: SysCatalogTable state changed. Reason: New leader bf9e588597644890990196db87d21a52. Latest consensus state: current_term: 1 leader_uuid: "bf9e588597644890990196db87d21a52" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81807fdfb68d46dc94b3a84d3f539416" member_type: VOTER last_known_addr { host: "127.4.238.62" port: 45103 } } peers { permanent_uuid: "8c99478dab94407b8f06ab5b292ab0d4" member_type: VOTER last_known_addr { host: "127.4.238.61" port: 41491 } } peers { permanent_uuid: "bf9e588597644890990196db87d21a52" member_type: VOTER last_known_addr { host: "127.4.238.60" port: 33899 } } }
21:12:54.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.575116 5295 sys_catalog.cc:458] T 00000000000000000000000000000000 P bf9e588597644890990196db87d21a52 [sys.catalog]: This master's current role is: LEADER
21:12:54.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.578701 5325 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
21:12:54.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.585475 5325 catalog_manager.cc:1494] Initializing Kudu cluster ID...
21:12:54.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.591352 5203 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8c99478dab94407b8f06ab5b292ab0d4: Bootstrap starting.
21:12:54.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.599109 5203 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 8c99478dab94407b8f06ab5b292ab0d4: Neither blocks nor log segments found. Creating new log.
21:12:54.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.601877 5203 log.cc:826] T 00000000000000000000000000000000 P 8c99478dab94407b8f06ab5b292ab0d4: Log is configured to *not* fsync() on all Append() calls
21:12:54.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.613523 5203 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8c99478dab94407b8f06ab5b292ab0d4: No bootstrap required, opened a new log
21:12:54.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.613680 5103 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 81807fdfb68d46dc94b3a84d3f539416 [term 1 FOLLOWER]: Refusing update from remote peer bf9e588597644890990196db87d21a52: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:12:54.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.615715 5321 consensus_queue.cc:1048] T 00000000000000000000000000000000 P bf9e588597644890990196db87d21a52 [LEADER]: Connected to new peer: Peer: permanent_uuid: "81807fdfb68d46dc94b3a84d3f539416" member_type: VOTER last_known_addr { host: "127.4.238.62" port: 45103 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:12:54.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.645584 5203 raft_consensus.cc:359] T 00000000000000000000000000000000 P 8c99478dab94407b8f06ab5b292ab0d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81807fdfb68d46dc94b3a84d3f539416" member_type: VOTER last_known_addr { host: "127.4.238.62" port: 45103 } } peers { permanent_uuid: "8c99478dab94407b8f06ab5b292ab0d4" member_type: VOTER last_known_addr { host: "127.4.238.61" port: 41491 } } peers { permanent_uuid: "bf9e588597644890990196db87d21a52" member_type: VOTER last_known_addr { host: "127.4.238.60" port: 33899 } }
21:12:54.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.646473 5203 raft_consensus.cc:385] T 00000000000000000000000000000000 P 8c99478dab94407b8f06ab5b292ab0d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:12:54.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.646764 5203 raft_consensus.cc:740] T 00000000000000000000000000000000 P 8c99478dab94407b8f06ab5b292ab0d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8c99478dab94407b8f06ab5b292ab0d4, State: Initialized, Role: FOLLOWER
21:12:54.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.647599 5203 consensus_queue.cc:260] T 00000000000000000000000000000000 P 8c99478dab94407b8f06ab5b292ab0d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81807fdfb68d46dc94b3a84d3f539416" member_type: VOTER last_known_addr { host: "127.4.238.62" port: 45103 } } peers { permanent_uuid: "8c99478dab94407b8f06ab5b292ab0d4" member_type: VOTER last_known_addr { host: "127.4.238.61" port: 41491 } } peers { permanent_uuid: "bf9e588597644890990196db87d21a52" member_type: VOTER last_known_addr { host: "127.4.238.60" port: 33899 } }
21:12:54.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.650235 5330 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8c99478dab94407b8f06ab5b292ab0d4 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81807fdfb68d46dc94b3a84d3f539416" member_type: VOTER last_known_addr { host: "127.4.238.62" port: 45103 } } peers { permanent_uuid: "8c99478dab94407b8f06ab5b292ab0d4" member_type: VOTER last_known_addr { host: "127.4.238.61" port: 41491 } } peers { permanent_uuid: "bf9e588597644890990196db87d21a52" member_type: VOTER last_known_addr { host: "127.4.238.60" port: 33899 } } }
21:12:54.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.657191 5296 sys_catalog.cc:455] T 00000000000000000000000000000000 P 81807fdfb68d46dc94b3a84d3f539416 [sys.catalog]: SysCatalogTable state changed. Reason: New leader bf9e588597644890990196db87d21a52. Latest consensus state: current_term: 1 leader_uuid: "bf9e588597644890990196db87d21a52" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81807fdfb68d46dc94b3a84d3f539416" member_type: VOTER last_known_addr { host: "127.4.238.62" port: 45103 } } peers { permanent_uuid: "8c99478dab94407b8f06ab5b292ab0d4" member_type: VOTER last_known_addr { host: "127.4.238.61" port: 41491 } } peers { permanent_uuid: "bf9e588597644890990196db87d21a52" member_type: VOTER last_known_addr { host: "127.4.238.60" port: 33899 } } }
21:12:54.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.657857 5296 sys_catalog.cc:458] T 00000000000000000000000000000000 P 81807fdfb68d46dc94b3a84d3f539416 [sys.catalog]: This master's current role is: FOLLOWER
21:12:54.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.663653 5330 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8c99478dab94407b8f06ab5b292ab0d4 [sys.catalog]: This master's current role is: FOLLOWER
21:12:54.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.652484 5177 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "bf9e588597644890990196db87d21a52" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8c99478dab94407b8f06ab5b292ab0d4"
21:12:54.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.653093 5178 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "bf9e588597644890990196db87d21a52" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8c99478dab94407b8f06ab5b292ab0d4" is_pre_election: true
21:12:54.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.654091 5176 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 8c99478dab94407b8f06ab5b292ab0d4 [term 0 FOLLOWER]: Advancing to term 1
21:12:54.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.663925 5296 sys_catalog.cc:455] T 00000000000000000000000000000000 P 81807fdfb68d46dc94b3a84d3f539416 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "bf9e588597644890990196db87d21a52" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81807fdfb68d46dc94b3a84d3f539416" member_type: VOTER last_known_addr { host: "127.4.238.62" port: 45103 } } peers { permanent_uuid: "8c99478dab94407b8f06ab5b292ab0d4" member_type: VOTER last_known_addr { host: "127.4.238.61" port: 41491 } } peers { permanent_uuid: "bf9e588597644890990196db87d21a52" member_type: VOTER last_known_addr { host: "127.4.238.60" port: 33899 } } }
21:12:54.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.667235 5296 sys_catalog.cc:458] T 00000000000000000000000000000000 P 81807fdfb68d46dc94b3a84d3f539416 [sys.catalog]: This master's current role is: FOLLOWER
21:12:54.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.667105 5322 sys_catalog.cc:455] T 00000000000000000000000000000000 P bf9e588597644890990196db87d21a52 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "bf9e588597644890990196db87d21a52" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81807fdfb68d46dc94b3a84d3f539416" member_type: VOTER last_known_addr { host: "127.4.238.62" port: 45103 } } peers { permanent_uuid: "8c99478dab94407b8f06ab5b292ab0d4" member_type: VOTER last_known_addr { host: "127.4.238.61" port: 41491 } } peers { permanent_uuid: "bf9e588597644890990196db87d21a52" member_type: VOTER last_known_addr { host: "127.4.238.60" port: 33899 } } }
21:12:54.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.667739 5322 sys_catalog.cc:458] T 00000000000000000000000000000000 P bf9e588597644890990196db87d21a52 [sys.catalog]: This master's current role is: LEADER
21:12:54.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.670532 5328 mvcc.cc:204] Tried to move back new op lower bound from 7231437105591115776 to 7231437105349169152. Current Snapshot: MvccSnapshot[applied={T|T < 7231437105591115776}]
21:12:54.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.652341 5203 sys_catalog.cc:565] T 00000000000000000000000000000000 P 8c99478dab94407b8f06ab5b292ab0d4 [sys.catalog]: configured and running, proceeding with master startup.
21:12:54.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.672371 5176 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 8c99478dab94407b8f06ab5b292ab0d4 [term 1 FOLLOWER]: Refusing update from remote peer bf9e588597644890990196db87d21a52: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:12:54.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.673929 5295 consensus_queue.cc:1048] T 00000000000000000000000000000000 P bf9e588597644890990196db87d21a52 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8c99478dab94407b8f06ab5b292ab0d4" member_type: VOTER last_known_addr { host: "127.4.238.61" port: 41491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:12:54.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.678010 5325 catalog_manager.cc:1357] Generated new cluster ID: 8d1cd2489534492aa80fb380ccda16ba
21:12:54.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.678279 5325 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
21:12:54.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.703200 5325 catalog_manager.cc:1380] Generated new certificate authority record
21:12:54.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.705826 5325 catalog_manager.cc:1514] Loading token signing keys...
21:12:54.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.731801 5295 sys_catalog.cc:455] T 00000000000000000000000000000000 P bf9e588597644890990196db87d21a52 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "bf9e588597644890990196db87d21a52" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81807fdfb68d46dc94b3a84d3f539416" member_type: VOTER last_known_addr { host: "127.4.238.62" port: 45103 } } peers { permanent_uuid: "8c99478dab94407b8f06ab5b292ab0d4" member_type: VOTER last_known_addr { host: "127.4.238.61" port: 41491 } } peers { permanent_uuid: "bf9e588597644890990196db87d21a52" member_type: VOTER last_known_addr { host: "127.4.238.60" port: 33899 } } }
21:12:54.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.732560 5295 sys_catalog.cc:458] T 00000000000000000000000000000000 P bf9e588597644890990196db87d21a52 [sys.catalog]: This master's current role is: LEADER
21:12:54.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.746676 5325 catalog_manager.cc:6027] T 00000000000000000000000000000000 P bf9e588597644890990196db87d21a52: Generated new TSK 0
21:12:54.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.747776 5325 catalog_manager.cc:1524] Initializing in-progress tserver states...
21:12:54.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.753347 5330 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8c99478dab94407b8f06ab5b292ab0d4 [sys.catalog]: SysCatalogTable state changed. Reason: New leader bf9e588597644890990196db87d21a52. Latest consensus state: current_term: 1 leader_uuid: "bf9e588597644890990196db87d21a52" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81807fdfb68d46dc94b3a84d3f539416" member_type: VOTER last_known_addr { host: "127.4.238.62" port: 45103 } } peers { permanent_uuid: "8c99478dab94407b8f06ab5b292ab0d4" member_type: VOTER last_known_addr { host: "127.4.238.61" port: 41491 } } peers { permanent_uuid: "bf9e588597644890990196db87d21a52" member_type: VOTER last_known_addr { host: "127.4.238.60" port: 33899 } } }
21:12:54.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.754557 5330 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8c99478dab94407b8f06ab5b292ab0d4 [sys.catalog]: This master's current role is: FOLLOWER
21:12:54.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.766048 5337 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8c99478dab94407b8f06ab5b292ab0d4 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "bf9e588597644890990196db87d21a52" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81807fdfb68d46dc94b3a84d3f539416" member_type: VOTER last_known_addr { host: "127.4.238.62" port: 45103 } } peers { permanent_uuid: "8c99478dab94407b8f06ab5b292ab0d4" member_type: VOTER last_known_addr { host: "127.4.238.61" port: 41491 } } peers { permanent_uuid: "bf9e588597644890990196db87d21a52" member_type: VOTER last_known_addr { host: "127.4.238.60" port: 33899 } } }
21:12:54.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.767977 5048 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu
21:12:54.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu
21:12:54.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-0/wal
21:12:54.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-0/data
21:12:54.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:12:54.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:12:54.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:12:54.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:12:54.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:12:54.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:12:54.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-0/logs
21:12:54.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-0/data/info.pb
21:12:54.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:12:54.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:12:54.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:12:54.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:12:54.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:12:54.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:12:54.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:12:54.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:12:54.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:12:54.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:12:54.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.238.1:0
21:12:54.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.238.1
21:12:54.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.238.1
21:12:54.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:12:54.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.238.62:45103,127.4.238.61:41491,127.4.238.60:33899
21:12:54.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.238.20:36399
21:12:54.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:12:54.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:12:54.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:12:54.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.773388 5337 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8c99478dab94407b8f06ab5b292ab0d4 [sys.catalog]: This master's current role is: FOLLOWER
21:12:54.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.926568 5350 catalog_manager.cc:1269] Loaded cluster ID: 8d1cd2489534492aa80fb380ccda16ba
21:12:54.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.927070 5350 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 8c99478dab94407b8f06ab5b292ab0d4: loading cluster ID for follower catalog manager: success
21:12:54.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.934391 5350 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 8c99478dab94407b8f06ab5b292ab0d4: acquiring CA information for follower catalog manager: success
21:12:54.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:54.940039 5350 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 8c99478dab94407b8f06ab5b292ab0d4: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:12:55.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:55.259120 5341 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:12:55.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:55.259747 5341 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:12:55.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:55.260524 5341 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:12:55.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:55.262540 5318 catalog_manager.cc:1269] Loaded cluster ID: 8d1cd2489534492aa80fb380ccda16ba
21:12:55.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:55.262915 5318 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 81807fdfb68d46dc94b3a84d3f539416: loading cluster ID for follower catalog manager: success
21:12:55.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:55.273376 5318 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 81807fdfb68d46dc94b3a84d3f539416: acquiring CA information for follower catalog manager: success
21:12:55.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:55.292898 5318 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 81807fdfb68d46dc94b3a84d3f539416: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:12:55.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:55.320298 5341 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:12:55.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:55.321679 5341 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.4.238.1
21:12:55.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:55.395120 5341 tablet_server_runner.cc:78] Tablet server non-default flags:
21:12:55.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:12:55.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.238.20:36399
21:12:55.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:12:55.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:12:55.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-0/data
21:12:55.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-0/wal
21:12:55.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:12:55.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:12:55.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.238.1:0
21:12:55.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:12:55.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:12:55.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:12:55.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-0/data/info.pb
21:12:55.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.238.1
21:12:55.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:12:55.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.238.62:45103,127.4.238.61:41491,127.4.238.60:33899
21:12:55.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:12:55.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:12:55.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:12:55.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:12:55.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:12:55.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.238.1
21:12:55.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-0/logs
21:12:55.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:12:55.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:12:55.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:12:55.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:12:55.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:12:55.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
21:12:55.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:12:55.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Dec 2025 20:43:24 UTC on 5fd53c4cbb9d
21:12:55.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9469
21:12:55.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:12:55.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:55.396919 5341 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:12:55.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:55.399078 5341 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:12:55.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:55.441257 5357 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:12:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:56.833210 5356 debug-util.cc:398] Leaking SignalData structure 0x7b080002ec60 after lost signal to thread 5341
21:12:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:57.707909 5341 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.258s user 0.823s sys 1.261s
21:12:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:57.708282 5341 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.259s user 0.823s sys 1.261s
21:12:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:55.451303 5358 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:12:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:57.716346 5341 server_base.cc:1047] running on GCE node
21:12:57.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:57.716893 5362 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:12:57.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:57.718240 5341 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:12:57.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:57.720701 5341 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:12:57.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:57.722121 5341 hybrid_clock.cc:648] HybridClock initialized: now 1765487577722084 us; error 25 us; skew 500 ppm
21:12:57.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:57.723183 5341 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:12:57.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:57.730388 5341 webserver.cc:492] Webserver started at http://127.4.238.1:37699/ using document root <none> and password file <none>
21:12:57.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:57.731586 5341 fs_manager.cc:362] Metadata directory not provided
21:12:57.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:57.731853 5341 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:12:57.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:57.732406 5341 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:12:57.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:57.738569 5341 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-0/data/instance:
21:12:57.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "77afd2665da54f71821b9b6077681447"
21:12:57.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-11 21:12:57 on dist-test-slave-2lgj"
21:12:57.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:57.740024 5341 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-0/wal/instance:
21:12:57.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "77afd2665da54f71821b9b6077681447"
21:12:57.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-11 21:12:57 on dist-test-slave-2lgj"
21:12:57.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:57.749083 5341 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.009s sys 0.001s
21:12:57.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:57.756317 5368 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:12:57.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:57.757582 5341 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
21:12:57.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:57.757997 5341 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-0/data,/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-0/wal
21:12:57.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "77afd2665da54f71821b9b6077681447"
21:12:57.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-11 21:12:57 on dist-test-slave-2lgj"
21:12:57.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:57.758411 5341 fs_report.cc:389] FS layout report
21:12:57.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:12:57.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-0/wal
21:12:57.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-0/wal
21:12:57.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-0/data/data
21:12:57.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:12:57.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:12:57.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:12:57.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:12:57.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:12:57.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:12:57.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:12:57.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:12:57.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:12:57.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:12:57.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:12:57.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:57.844584 5341 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:12:57.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:57.846035 5341 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:12:57.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:57.846427 5341 kserver.cc:163] Server-wide thread pool size limit: 3276
21:12:57.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:57.849231 5341 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:12:57.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:57.853662 5341 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:12:57.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:57.853900 5341 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.001s sys 0.000s
21:12:57.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:57.854184 5341 ts_tablet_manager.cc:616] Registered 0 tablets
21:12:57.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:57.854346 5341 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:12:58.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:58.009123 5341 rpc_server.cc:307] RPC server started. Bound to: 127.4.238.1:34441
21:12:58.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:58.009258 5480 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.238.1:34441 every 8 connection(s)
21:12:58.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:58.011813 5341 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-0/data/info.pb
21:12:58.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:58.021128 5048 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu as pid 5341
21:12:58.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:58.021565 5048 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-0/wal/instance
21:12:58.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:58.040426 5048 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu
21:12:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu
21:12:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-1/wal
21:12:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-1/data
21:12:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:12:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:12:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:12:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:12:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:12:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:12:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-1/logs
21:12:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-1/data/info.pb
21:12:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:12:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:12:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:12:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:12:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:12:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:12:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:12:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:12:58.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:12:58.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:12:58.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.238.2:0
21:12:58.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.238.2
21:12:58.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.238.2
21:12:58.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:12:58.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.238.62:45103,127.4.238.61:41491,127.4.238.60:33899
21:12:58.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.238.20:36399
21:12:58.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:12:58.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:12:58.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:12:58.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:58.075973 5481 heartbeater.cc:344] Connected to a master server at 127.4.238.60:33899
21:12:58.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:58.076537 5481 heartbeater.cc:461] Registering TS with master...
21:12:58.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:58.077915 5481 heartbeater.cc:507] Master 127.4.238.60:33899 requested a full tablet report, sending...
21:12:58.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:58.078485 5483 heartbeater.cc:344] Connected to a master server at 127.4.238.62:45103
21:12:58.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:58.078848 5483 heartbeater.cc:461] Registering TS with master...
21:12:58.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:58.079770 5483 heartbeater.cc:507] Master 127.4.238.62:45103 requested a full tablet report, sending...
21:12:58.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:58.081001 5482 heartbeater.cc:344] Connected to a master server at 127.4.238.61:41491
21:12:58.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:58.080956 5248 ts_manager.cc:194] Registered new tserver with Master: 77afd2665da54f71821b9b6077681447 (127.4.238.1:34441)
21:12:58.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:58.081347 5482 heartbeater.cc:461] Registering TS with master...
21:12:58.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:58.082227 5482 heartbeater.cc:507] Master 127.4.238.61:41491 requested a full tablet report, sending...
21:12:58.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:58.084026 5093 ts_manager.cc:194] Registered new tserver with Master: 77afd2665da54f71821b9b6077681447 (127.4.238.1:34441)
21:12:58.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:58.084061 5248 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.4.238.1:49005
21:12:58.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:58.085182 5168 ts_manager.cc:194] Registered new tserver with Master: 77afd2665da54f71821b9b6077681447 (127.4.238.1:34441)
21:12:58.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:58.409224 5491 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:12:58.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:58.409706 5491 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:12:58.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:58.410202 5491 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:12:58.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:58.442452 5491 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:12:58.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:58.443310 5491 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.4.238.2
21:12:58.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:58.477528 5491 tablet_server_runner.cc:78] Tablet server non-default flags:
21:12:58.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:12:58.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.238.20:36399
21:12:58.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:12:58.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:12:58.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-1/data
21:12:58.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-1/wal
21:12:58.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:12:58.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:12:58.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.238.2:0
21:12:58.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:12:58.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:12:58.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:12:58.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-1/data/info.pb
21:12:58.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.238.2
21:12:58.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:12:58.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.238.62:45103,127.4.238.61:41491,127.4.238.60:33899
21:12:58.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:12:58.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:12:58.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:12:58.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:12:58.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:12:58.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.238.2
21:12:58.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-1/logs
21:12:58.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:12:58.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:12:58.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:12:58.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:12:58.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:12:58.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
21:12:58.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:12:58.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Dec 2025 20:43:24 UTC on 5fd53c4cbb9d
21:12:58.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9469
21:12:58.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:12:58.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:58.478866 5491 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:12:58.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:58.480392 5491 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:12:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:58.492393 5498 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:12:59.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:59.088169 5481 heartbeater.cc:499] Master 127.4.238.60:33899 was elected leader, sending a full tablet report...
21:12:59.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:59.895380 5497 debug-util.cc:398] Leaking SignalData structure 0x7b080002ec60 after lost signal to thread 5491
21:12:59.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:59.988382 5497 kernel_stack_watchdog.cc:198] Thread 5491 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 399ms:
21:12:59.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
21:12:59.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
21:12:59.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
21:12:59.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
21:12:59.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:58.493958 5499 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:12:59.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:59.989689 5491 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.496s user 0.000s sys 0.002s
21:12:59.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:59.990025 5491 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.497s user 0.000s sys 0.002s
21:12:59.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:59.993206 5500 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1495 milliseconds
21:12:59.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:12:59.994560 5501 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:12:59.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:59.994632 5491 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:12:59.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:59.995915 5491 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:12:59.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:59.998291 5491 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:12:59.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:12:59.999687 5491 hybrid_clock.cc:648] HybridClock initialized: now 1765487579999631 us; error 56 us; skew 500 ppm
21:13:00.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:00.000517 5491 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:13:00.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:00.006795 5491 webserver.cc:492] Webserver started at http://127.4.238.2:41095/ using document root <none> and password file <none>
21:13:00.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:00.007663 5491 fs_manager.cc:362] Metadata directory not provided
21:13:00.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:00.007856 5491 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:13:00.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:00.008237 5491 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:13:00.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:00.012466 5491 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-1/data/instance:
21:13:00.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eea63386f1d34759b7a1492790163294"
21:13:00.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-11 21:13:00 on dist-test-slave-2lgj"
21:13:00.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:00.013520 5491 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-1/wal/instance:
21:13:00.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eea63386f1d34759b7a1492790163294"
21:13:00.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-11 21:13:00 on dist-test-slave-2lgj"
21:13:00.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:00.020558 5491 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.001s sys 0.004s
21:13:00.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:00.025936 5510 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:13:00.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:00.026921 5491 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
21:13:00.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:00.027217 5491 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-1/data,/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-1/wal
21:13:00.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eea63386f1d34759b7a1492790163294"
21:13:00.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-11 21:13:00 on dist-test-slave-2lgj"
21:13:00.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:00.027500 5491 fs_report.cc:389] FS layout report
21:13:00.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:13:00.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-1/wal
21:13:00.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-1/wal
21:13:00.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-1/data/data
21:13:00.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:13:00.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:13:00.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:13:00.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:13:00.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:13:00.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:13:00.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:13:00.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:13:00.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:13:00.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:13:00.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:13:00.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:00.073195 5491 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:13:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:00.074672 5491 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:13:00.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:00.075064 5491 kserver.cc:163] Server-wide thread pool size limit: 3276
21:13:00.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:00.077472 5491 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:13:00.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:00.081765 5491 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:13:00.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:00.081991 5491 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.001s
21:13:00.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:00.082188 5491 ts_tablet_manager.cc:616] Registered 0 tablets
21:13:00.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:00.082321 5491 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:13:00.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:00.217401 5491 rpc_server.cc:307] RPC server started. Bound to: 127.4.238.2:39163
21:13:00.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:00.217502 5622 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.238.2:39163 every 8 connection(s)
21:13:00.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:00.219911 5491 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-1/data/info.pb
21:13:00.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:00.228814 5048 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu as pid 5491
21:13:00.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:00.229439 5048 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-1/wal/instance
21:13:00.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:00.244776 5048 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu
21:13:00.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu
21:13:00.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-2/wal
21:13:00.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-2/data
21:13:00.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:13:00.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:13:00.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:13:00.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:13:00.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:13:00.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:13:00.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-2/logs
21:13:00.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-2/data/info.pb
21:13:00.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:13:00.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:13:00.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:13:00.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:13:00.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:13:00.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:13:00.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:13:00.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:13:00.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:13:00.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:13:00.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.238.3:0
21:13:00.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.238.3
21:13:00.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.238.3
21:13:00.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:13:00.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.238.62:45103,127.4.238.61:41491,127.4.238.60:33899
21:13:00.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.238.20:36399
21:13:00.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:13:00.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:13:00.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:13:00.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:00.250828 5623 heartbeater.cc:344] Connected to a master server at 127.4.238.60:33899
21:13:00.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:00.251346 5623 heartbeater.cc:461] Registering TS with master...
21:13:00.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:00.252673 5623 heartbeater.cc:507] Master 127.4.238.60:33899 requested a full tablet report, sending...
21:13:00.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:00.255688 5248 ts_manager.cc:194] Registered new tserver with Master: eea63386f1d34759b7a1492790163294 (127.4.238.2:39163)
21:13:00.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:00.257283 5248 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.4.238.2:51989
21:13:00.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:00.259012 5624 heartbeater.cc:344] Connected to a master server at 127.4.238.61:41491
21:13:00.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:00.259328 5624 heartbeater.cc:461] Registering TS with master...
21:13:00.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:00.259866 5625 heartbeater.cc:344] Connected to a master server at 127.4.238.62:45103
21:13:00.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:00.260123 5625 heartbeater.cc:461] Registering TS with master...
21:13:00.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:00.260471 5624 heartbeater.cc:507] Master 127.4.238.61:41491 requested a full tablet report, sending...
21:13:00.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:00.260635 5625 heartbeater.cc:507] Master 127.4.238.62:45103 requested a full tablet report, sending...
21:13:00.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:00.262989 5168 ts_manager.cc:194] Registered new tserver with Master: eea63386f1d34759b7a1492790163294 (127.4.238.2:39163)
21:13:00.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:00.262969 5093 ts_manager.cc:194] Registered new tserver with Master: eea63386f1d34759b7a1492790163294 (127.4.238.2:39163)
21:13:00.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:00.540776 5633 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:13:00.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:00.541308 5633 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:13:00.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:00.541870 5633 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:13:00.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:00.572463 5633 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:13:00.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:00.573328 5633 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.4.238.3
21:13:00.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:00.606642 5633 tablet_server_runner.cc:78] Tablet server non-default flags:
21:13:00.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:13:00.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.238.20:36399
21:13:00.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:13:00.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:13:00.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-2/data
21:13:00.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-2/wal
21:13:00.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:13:00.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:13:00.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.238.3:0
21:13:00.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:13:00.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:13:00.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:13:00.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-2/data/info.pb
21:13:00.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.238.3
21:13:00.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:13:00.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.238.62:45103,127.4.238.61:41491,127.4.238.60:33899
21:13:00.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:13:00.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:13:00.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:13:00.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:13:00.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:13:00.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.238.3
21:13:00.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-2/logs
21:13:00.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:13:00.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:13:00.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:13:00.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:13:00.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:13:00.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
21:13:00.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:13:00.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Dec 2025 20:43:24 UTC on 5fd53c4cbb9d
21:13:00.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9469
21:13:00.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:13:00.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:00.608124 5633 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:13:00.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:00.609607 5633 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:13:00.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:00.620838 5639 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:13:01.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:01.261435 5623 heartbeater.cc:499] Master 127.4.238.60:33899 was elected leader, sending a full tablet report...
21:13:02.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:02.022970 5638 debug-util.cc:398] Leaking SignalData structure 0x7b080002ec60 after lost signal to thread 5633
21:13:02.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:00.622195 5640 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:13:02.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:02.144287 5633 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.522s user 0.608s sys 0.913s
21:13:02.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:02.145987 5633 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.524s user 0.608s sys 0.913s
21:13:02.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:02.146624 5642 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:13:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:02.147140 5641 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1523 milliseconds
21:13:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.147084 5633 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:13:02.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.150243 5633 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:13:02.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.152154 5633 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:13:02.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.153465 5633 hybrid_clock.cc:648] HybridClock initialized: now 1765487582153440 us; error 35 us; skew 500 ppm
21:13:02.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.154253 5633 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:13:02.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.160033 5633 webserver.cc:492] Webserver started at http://127.4.238.3:34207/ using document root <none> and password file <none>
21:13:02.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.160878 5633 fs_manager.cc:362] Metadata directory not provided
21:13:02.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.161077 5633 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:13:02.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.161449 5633 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:13:02.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.165422 5633 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-2/data/instance:
21:13:02.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "01ac729fa9b440c8bc381d80cc3ad78d"
21:13:02.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-11 21:13:02 on dist-test-slave-2lgj"
21:13:02.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.166431 5633 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-2/wal/instance:
21:13:02.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "01ac729fa9b440c8bc381d80cc3ad78d"
21:13:02.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-11 21:13:02 on dist-test-slave-2lgj"
21:13:02.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.173118 5633 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.001s sys 0.005s
21:13:02.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.178458 5650 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:13:02.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.179431 5633 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.002s
21:13:02.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.179688 5633 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-2/data,/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-2/wal
21:13:02.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "01ac729fa9b440c8bc381d80cc3ad78d"
21:13:02.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-11 21:13:02 on dist-test-slave-2lgj"
21:13:02.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.179968 5633 fs_report.cc:389] FS layout report
21:13:02.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:13:02.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-2/wal
21:13:02.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-2/wal
21:13:02.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-2/data/data
21:13:02.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:13:02.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:13:02.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:13:02.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:13:02.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:13:02.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:13:02.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:13:02.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:13:02.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:13:02.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:13:02.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:13:02.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.237634 5633 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:13:02.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.239049 5633 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:13:02.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.239428 5633 kserver.cc:163] Server-wide thread pool size limit: 3276
21:13:02.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.241760 5633 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:13:02.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.245642 5633 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:13:02.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.245852 5633 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.001s
21:13:02.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.246044 5633 ts_tablet_manager.cc:616] Registered 0 tablets
21:13:02.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.246192 5633 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:13:02.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.370713 5633 rpc_server.cc:307] RPC server started. Bound to: 127.4.238.3:34817
21:13:02.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.370848 5762 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.238.3:34817 every 8 connection(s)
21:13:02.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.373224 5633 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-2/data/info.pb
21:13:02.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.383019 5048 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu as pid 5633
21:13:02.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.383497 5048 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster8983971028589054682/ts-2/wal/instance
21:13:02.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.415606 5764 heartbeater.cc:344] Connected to a master server at 127.4.238.61:41491
21:13:02.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.415906 5763 heartbeater.cc:344] Connected to a master server at 127.4.238.60:33899
21:13:02.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.416114 5764 heartbeater.cc:461] Registering TS with master...
21:13:02.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.416627 5765 heartbeater.cc:344] Connected to a master server at 127.4.238.62:45103
21:13:02.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.416671 5763 heartbeater.cc:461] Registering TS with master...
21:13:02.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.417068 5765 heartbeater.cc:461] Registering TS with master...
21:13:02.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.417375 5764 heartbeater.cc:507] Master 127.4.238.61:41491 requested a full tablet report, sending...
21:13:02.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.417882 5763 heartbeater.cc:507] Master 127.4.238.60:33899 requested a full tablet report, sending...
21:13:02.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.417943 5765 heartbeater.cc:507] Master 127.4.238.62:45103 requested a full tablet report, sending...
21:13:02.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.420269 5248 ts_manager.cc:194] Registered new tserver with Master: 01ac729fa9b440c8bc381d80cc3ad78d (127.4.238.3:34817)
21:13:02.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.420320 5168 ts_manager.cc:194] Registered new tserver with Master: 01ac729fa9b440c8bc381d80cc3ad78d (127.4.238.3:34817)
21:13:02.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.420497 5093 ts_manager.cc:194] Registered new tserver with Master: 01ac729fa9b440c8bc381d80cc3ad78d (127.4.238.3:34817)
21:13:02.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.421918 5248 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.4.238.3:34643
21:13:02.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.437927 5048 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
21:13:02.448 [DEBUG - main] (MiniKuduCluster.java:195) Response:
21:13:02.448 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
21:13:02.449 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.4.238.62"
port: 45103
}
bound_http_address {
host: "127.4.238.62"
port: 43101
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.4.238.61"
port: 41491
}
bound_http_address {
host: "127.4.238.61"
port: 33921
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.4.238.60"
port: 33899
}
bound_http_address {
host: "127.4.238.60"
port: 45651
}
}
}
21:13:02.449 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
21:13:02.450 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.4.238.1"
port: 34441
}
bound_http_address {
host: "127.4.238.1"
port: 37699
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.4.238.2"
port: 39163
}
bound_http_address {
host: "127.4.238.2"
port: 41095
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.4.238.3"
port: 34817
}
bound_http_address {
host: "127.4.238.3"
port: 34207
}
}
}
21:13:02.450 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
21:13:02.450 [DEBUG - main] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster851002246870464567/krb5kdc/krb5cc
21:13:02.451 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
21:13:02.452 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskpJdSDR/test-tmp
21:13:02.452 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
21:13:02.452 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskpJdSDR/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
21:13:02.453 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
21:13:02.775 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:13:02.776 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
21:13:02.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.779517 5774 mini_oidc.cc:150] Starting JWKS server
21:13:02.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.782424 5774 webserver.cc:492] Webserver started at http://127.0.0.1:43381/ using document root <none> and password file <none>
21:13:02.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.782809 5774 mini_oidc.cc:186] Starting OIDC Discovery server
21:13:02.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:02.784366 5774 webserver.cc:492] Webserver started at http://127.0.0.1:38531/ using document root <none> and password file <none>
21:13:02.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
21:13:02.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/principal' for realm 'KRBTEST.COM',
21:13:02.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
21:13:02.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 11 21:13:02 dist-test-slave-2lgj krb5kdc[5786](info): setting up network...
21:13:02.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
21:13:02.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 11 21:13:02 dist-test-slave-2lgj krb5kdc[5786](info): set up 2 sockets
21:13:02.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 11 21:13:02 dist-test-slave-2lgj krb5kdc[5786](info): commencing operation
21:13:02.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
21:13:03.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:03.424824 5763 heartbeater.cc:499] Master 127.4.238.60:33899 was elected leader, sending a full tablet report...
21:13:04.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:04.696980 5774 mini_kdc.cc:121] Time spent starting KDC: real 1.879s user 0.009s sys 0.008s
21:13:04.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
21:13:04.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
21:13:04.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
21:13:04.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
21:13:04.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
21:13:04.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
21:13:04.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
21:13:04.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
21:13:04.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
21:13:04.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
21:13:04.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/test-user.keytab.
21:13:04.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/test-user.keytab.
21:13:04.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 11 21:13:04 dist-test-slave-2lgj krb5kdc[5786](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765487584, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:13:04.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
21:13:04.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-11T21:13:04Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
21:13:04.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-11T21:13:04Z Disabled control of system clock
21:13:04.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.5.163.190@KRBTEST.COM; defaulting to no policy
21:13:04.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:13:04.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.5.163.190@KRBTEST.COM" created.
21:13:04.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:13:04.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.5.163.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/kudu.keytab.
21:13:04.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.5.163.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/kudu.keytab.
21:13:04.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.5.163.190@KRBTEST.COM; defaulting to no policy
21:13:04.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:13:04.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.5.163.190@KRBTEST.COM" created.
21:13:04.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:13:04.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.5.163.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/kudu.keytab.
21:13:04.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.5.163.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/kudu.keytab.
21:13:04.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:04.889920 5774 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu
21:13:04.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu
21:13:04.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-0/wal
21:13:04.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-0/data
21:13:04.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:13:04.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:13:04.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:13:04.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:13:04.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:13:04.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:13:04.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-0/logs
21:13:04.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-0/data/info.pb
21:13:04.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:13:04.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:13:04.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:13:04.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:13:04.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:13:04.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:13:04.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:13:04.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:13:04.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:13:04.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:13:04.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:13:04.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:13:04.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.163.190:35663
21:13:04.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.163.190
21:13:04.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:13:04.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/kudu.keytab
21:13:04.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.163.190
21:13:04.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:13:04.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:13:04.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:13:04.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:13:04.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.163.148:33837
21:13:04.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:13:04.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:13:04.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:13:04.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:13:04.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.163.190:35663,127.5.163.189:41991,127.5.163.188:32827
21:13:04.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:13:04.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43381/jwks.json
21:13:04.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:13:04.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:13:05.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:05.181777 5802 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:13:05.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:05.182294 5802 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:13:05.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:05.182904 5802 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:13:05.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:05.215765 5802 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:13:05.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:05.215999 5802 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:13:05.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:05.216303 5802 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:13:05.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:05.216511 5802 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:13:05.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:05.216701 5802 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:13:05.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:05.216904 5802 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:13:05.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:05.217056 5802 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43381/jwks.json
21:13:05.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:05.253074 5802 master_runner.cc:386] Master server non-default flags:
21:13:05.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:13:05.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.163.148:33837
21:13:05.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:13:05.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:13:05.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-0/data
21:13:05.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-0/wal
21:13:05.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:13:05.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:13:05.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.163.190:35663,127.5.163.189:41991,127.5.163.188:32827
21:13:05.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/kudu.keytab
21:13:05.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.163.190
21:13:05.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:13:05.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:13:05.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:13:05.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.163.190:35663
21:13:05.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:13:05.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:13:05.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:13:05.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43381/jwks.json
21:13:05.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:13:05.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:13:05.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:13:05.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:13:05.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:13:05.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-0/data/info.pb
21:13:05.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.163.190
21:13:05.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:13:05.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:13:05.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:13:05.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:13:05.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:13:05.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:13:05.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-0/logs
21:13:05.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:13:05.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:13:05.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:13:05.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:13:05.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:13:05.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
21:13:05.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:13:05.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Dec 2025 20:43:24 UTC on 5fd53c4cbb9d
21:13:05.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9469
21:13:05.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:13:05.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:05.254412 5802 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:13:05.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:05.255437 5802 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:13:05.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:05.266007 5809 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:13:06.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:05.266767 5810 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:13:06.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:06.643316 5812 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:13:06.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:06.647231 5802 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.381s user 0.553s sys 0.824s
21:13:06.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:06.647464 5811 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1379 milliseconds
21:13:06.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:06.647579 5802 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.382s user 0.553s sys 0.824s
21:13:06.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:06.647850 5802 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:13:06.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:06.648916 5802 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:13:06.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:06.651356 5802 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:13:06.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:06.652695 5802 hybrid_clock.cc:648] HybridClock initialized: now 1765487586652652 us; error 40 us; skew 500 ppm
21:13:06.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:06.653489 5802 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:13:06.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 11 21:13:06 dist-test-slave-2lgj krb5kdc[5786](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765487586, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.163.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:13:06.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:06.660122 5802 init.cc:377] Logged in from keytab as kudu/127.5.163.190@KRBTEST.COM (short username kudu)
21:13:06.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:06.665865 5802 webserver.cc:492] Webserver started at http://127.5.163.190:37105/ using document root <none> and password file <none>
21:13:06.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:06.667019 5802 fs_manager.cc:362] Metadata directory not provided
21:13:06.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:06.667285 5802 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:13:06.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:06.667857 5802 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:13:06.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:06.673817 5802 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-0/data/instance:
21:13:06.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dc7d413f28fc40929b53b58b87dc209e"
21:13:06.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-11 21:13:06 on dist-test-slave-2lgj"
21:13:06.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:06.675007 5802 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-0/wal/instance:
21:13:06.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dc7d413f28fc40929b53b58b87dc209e"
21:13:06.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-11 21:13:06 on dist-test-slave-2lgj"
21:13:06.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:06.682538 5802 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.004s sys 0.004s
21:13:06.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:06.688035 5820 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:13:06.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:06.689356 5802 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.003s
21:13:06.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:06.689708 5802 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-0/data,/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-0/wal
21:13:06.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dc7d413f28fc40929b53b58b87dc209e"
21:13:06.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-11 21:13:06 on dist-test-slave-2lgj"
21:13:06.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:06.690142 5802 fs_report.cc:389] FS layout report
21:13:06.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:13:06.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-0/wal
21:13:06.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-0/wal
21:13:06.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-0/data/data
21:13:06.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:13:06.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:13:06.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:13:06.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:13:06.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:13:06.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:13:06.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:13:06.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:13:06.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:13:06.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:13:06.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:13:06.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:06.759518 5802 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:13:06.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:06.767879 5802 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:13:06.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:06.768267 5802 kserver.cc:163] Server-wide thread pool size limit: 3276
21:13:06.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:06.835906 5802 rpc_server.cc:307] RPC server started. Bound to: 127.5.163.190:35663
21:13:06.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:06.835996 5871 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.5.163.190:35663 every 8 connection(s)
21:13:06.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:06.838435 5802 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-0/data/info.pb
21:13:06.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:06.840976 5774 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu as pid 5802
21:13:06.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:06.841523 5774 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-0/wal/instance
21:13:06.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:06.844354 5872 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:13:06.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.5.163.189@KRBTEST.COM; defaulting to no policy
21:13:06.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:06.857463 5872 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.5.163.190" port: 35663 } has no permanent_uuid. Determining permanent_uuid...
21:13:06.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:13:06.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.5.163.189@KRBTEST.COM" created.
21:13:06.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 11 21:13:06 dist-test-slave-2lgj krb5kdc[5786](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765487586, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.163.190@KRBTEST.COM for kudu/127.5.163.190@KRBTEST.COM
21:13:06.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:13:06.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.5.163.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/kudu.keytab.
21:13:06.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.5.163.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/kudu.keytab.
21:13:06.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:06.892958 5872 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.5.163.189" port: 41991 } has no permanent_uuid. Determining permanent_uuid...
21:13:06.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:06.895112 5822 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.5.163.189:41991: connect: Connection refused (error 111)
21:13:06.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.5.163.189@KRBTEST.COM; defaulting to no policy
21:13:06.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:13:06.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.5.163.189@KRBTEST.COM" created.
21:13:06.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:06.898664 5872 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.5.163.189:41991: Network error: Client connection negotiation failed: client connection to 127.5.163.189:41991: connect: Connection refused (error 111)
21:13:06.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:13:06.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.5.163.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/kudu.keytab.
21:13:06.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.5.163.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/kudu.keytab.
21:13:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:06.915647 5774 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu
21:13:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu
21:13:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-1/wal
21:13:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-1/data
21:13:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:13:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:13:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:13:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:13:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:13:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:13:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-1/logs
21:13:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-1/data/info.pb
21:13:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:13:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:13:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:13:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:13:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:13:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:13:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:13:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:13:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:13:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:13:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:13:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:13:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.163.189:41991
21:13:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.163.189
21:13:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:13:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/kudu.keytab
21:13:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.163.189
21:13:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:13:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:13:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:13:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:13:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.163.148:33837
21:13:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:13:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:13:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:13:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:13:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.163.190:35663,127.5.163.189:41991,127.5.163.188:32827
21:13:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:13:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43381/jwks.json
21:13:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:13:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:13:06.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:06.948145 5872 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.163.189" port: 41991 } attempt: 1
21:13:06.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:06.951624 5872 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.5.163.189:41991: Network error: Client connection negotiation failed: client connection to 127.5.163.189:41991: connect: Connection refused (error 111)
21:13:07.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:07.019970 5872 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.163.189" port: 41991 } attempt: 2
21:13:07.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:07.023712 5872 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.5.163.189:41991: Network error: Client connection negotiation failed: client connection to 127.5.163.189:41991: connect: Connection refused (error 111)
21:13:07.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:07.115077 5872 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.163.189" port: 41991 } attempt: 3
21:13:07.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:07.118990 5872 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.5.163.189:41991: Network error: Client connection negotiation failed: client connection to 127.5.163.189:41991: connect: Connection refused (error 111)
21:13:07.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:07.212994 5881 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:13:07.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:07.213502 5881 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:13:07.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:07.214151 5881 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:13:07.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:07.250442 5881 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:13:07.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:07.250662 5881 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:13:07.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:07.251022 5881 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:13:07.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:07.251242 5881 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:13:07.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:07.251425 5881 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:13:07.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:07.251603 5881 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:13:07.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:07.251750 5881 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43381/jwks.json
21:13:07.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:07.262434 5872 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.163.189" port: 41991 } attempt: 4
21:13:07.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:07.266366 5872 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.5.163.189:41991: Network error: Client connection negotiation failed: client connection to 127.5.163.189:41991: connect: Connection refused (error 111)
21:13:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:07.288697 5881 master_runner.cc:386] Master server non-default flags:
21:13:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:13:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.163.148:33837
21:13:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:13:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:13:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-1/data
21:13:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-1/wal
21:13:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:13:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:13:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.163.190:35663,127.5.163.189:41991,127.5.163.188:32827
21:13:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/kudu.keytab
21:13:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.163.189
21:13:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:13:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:13:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:13:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.163.189:41991
21:13:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:13:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:13:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:13:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43381/jwks.json
21:13:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:13:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:13:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:13:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:13:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:13:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-1/data/info.pb
21:13:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.163.189
21:13:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:13:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:13:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:13:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:13:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:13:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:13:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-1/logs
21:13:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:13:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:13:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:13:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:13:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:13:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
21:13:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:13:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Dec 2025 20:43:24 UTC on 5fd53c4cbb9d
21:13:07.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9469
21:13:07.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:13:07.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:07.289881 5881 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:13:07.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:07.290818 5881 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:13:07.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:07.303786 5888 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:13:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:07.565723 5872 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.163.189" port: 41991 } attempt: 5
21:13:07.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:07.573690 5872 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.5.163.189:41991: Network error: Client connection negotiation failed: client connection to 127.5.163.189:41991: connect: Connection refused (error 111)
21:13:08.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:08.121209 5872 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.163.189" port: 41991 } attempt: 6
21:13:08.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:08.171543 5872 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.5.163.189:41991: Network error: Client connection negotiation failed: client connection to 127.5.163.189:41991: connect: Connection refused (error 111)
21:13:08.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:08.707082 5886 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 5881
21:13:08.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:07.304042 5887 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:13:08.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:08.722677 5886 kernel_stack_watchdog.cc:198] Thread 5881 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
21:13:08.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
21:13:08.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
21:13:08.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
21:13:08.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
21:13:08.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:08.723147 5881 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.418s user 0.000s sys 0.002s
21:13:08.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:08.723564 5881 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.419s user 0.000s sys 0.002s
21:13:08.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:08.724159 5889 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1417 milliseconds
21:13:08.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:08.724972 5891 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:13:08.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:08.724952 5881 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:13:08.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:08.728495 5881 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:13:08.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:08.732151 5881 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:13:08.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:08.733530 5881 hybrid_clock.cc:648] HybridClock initialized: now 1765487588733491 us; error 44 us; skew 500 ppm
21:13:08.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:08.734336 5881 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:13:08.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 11 21:13:08 dist-test-slave-2lgj krb5kdc[5786](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765487588, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.163.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:13:08.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:08.739807 5881 init.cc:377] Logged in from keytab as kudu/127.5.163.189@KRBTEST.COM (short username kudu)
21:13:08.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:08.744765 5881 webserver.cc:492] Webserver started at http://127.5.163.189:34781/ using document root <none> and password file <none>
21:13:08.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:08.745580 5881 fs_manager.cc:362] Metadata directory not provided
21:13:08.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:08.745754 5881 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:13:08.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:08.746177 5881 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:13:08.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:08.750113 5881 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-1/data/instance:
21:13:08.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dc9b40a769d748be83c1ae82b0e2a748"
21:13:08.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-11 21:13:08 on dist-test-slave-2lgj"
21:13:08.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:08.751093 5881 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-1/wal/instance:
21:13:08.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dc9b40a769d748be83c1ae82b0e2a748"
21:13:08.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-11 21:13:08 on dist-test-slave-2lgj"
21:13:08.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:08.758317 5881 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
21:13:08.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:08.763600 5903 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:13:08.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:08.764542 5881 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.000s sys 0.005s
21:13:08.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:08.764853 5881 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-1/data,/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-1/wal
21:13:08.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dc9b40a769d748be83c1ae82b0e2a748"
21:13:08.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-11 21:13:08 on dist-test-slave-2lgj"
21:13:08.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:08.765153 5881 fs_report.cc:389] FS layout report
21:13:08.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:13:08.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-1/wal
21:13:08.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-1/wal
21:13:08.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-1/data/data
21:13:08.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:13:08.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:13:08.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:13:08.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:13:08.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:13:08.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:13:08.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:13:08.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:13:08.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:13:08.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:13:08.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:13:08.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:08.815449 5881 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:13:08.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:08.820653 5881 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:13:08.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:08.821090 5881 kserver.cc:163] Server-wide thread pool size limit: 3276
21:13:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:08.898874 5881 rpc_server.cc:307] RPC server started. Bound to: 127.5.163.189:41991
21:13:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:08.898947 5954 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.5.163.189:41991 every 8 connection(s)
21:13:08.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:08.901490 5881 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-1/data/info.pb
21:13:08.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:08.906810 5955 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:13:08.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:08.907810 5774 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu as pid 5881
21:13:08.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:08.908267 5774 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-1/wal/instance
21:13:08.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:08.919857 5955 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.5.163.190" port: 35663 } has no permanent_uuid. Determining permanent_uuid...
21:13:08.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.5.163.188@KRBTEST.COM; defaulting to no policy
21:13:08.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:13:08.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.5.163.188@KRBTEST.COM" created.
21:13:08.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 11 21:13:08 dist-test-slave-2lgj krb5kdc[5786](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765487588, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.163.189@KRBTEST.COM for kudu/127.5.163.190@KRBTEST.COM
21:13:08.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:13:08.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.5.163.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/kudu.keytab.
21:13:08.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.5.163.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/kudu.keytab.
21:13:08.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:08.956470 5955 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.5.163.189" port: 41991 } has no permanent_uuid. Determining permanent_uuid...
21:13:08.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.5.163.188@KRBTEST.COM; defaulting to no policy
21:13:08.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:13:08.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.5.163.188@KRBTEST.COM" created.
21:13:08.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 11 21:13:08 dist-test-slave-2lgj krb5kdc[5786](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765487588, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.163.189@KRBTEST.COM for kudu/127.5.163.189@KRBTEST.COM
21:13:08.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:08.980696 5955 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } has no permanent_uuid. Determining permanent_uuid...
21:13:08.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:08.982702 5906 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.5.163.188:32827: connect: Connection refused (error 111)
21:13:08.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:13:08.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.5.163.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/kudu.keytab.
21:13:08.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.5.163.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/kudu.keytab.
21:13:08.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:08.986181 5955 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.5.163.188:32827: Network error: Client connection negotiation failed: client connection to 127.5.163.188:32827: connect: Connection refused (error 111)
21:13:08.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:08.987867 5774 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu
21:13:08.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu
21:13:08.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-2/wal
21:13:08.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-2/data
21:13:08.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:13:08.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:13:08.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:13:08.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:13:08.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:13:08.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:13:08.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-2/logs
21:13:08.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-2/data/info.pb
21:13:08.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:13:08.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:13:08.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:13:08.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:13:08.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:13:08.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:13:08.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:13:08.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:13:08.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:13:08.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:13:08.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:13:08.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:13:08.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.163.188:32827
21:13:08.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.163.188
21:13:08.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:13:08.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/kudu.keytab
21:13:08.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.163.188
21:13:08.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:13:08.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:13:08.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:13:08.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:13:08.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.163.148:33837
21:13:08.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:13:08.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:13:08.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:13:08.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:13:08.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.163.190:35663,127.5.163.189:41991,127.5.163.188:32827
21:13:08.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:13:08.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43381/jwks.json
21:13:08.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:13:08.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:13:09.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:09.035686 5955 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } attempt: 1
21:13:09.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:09.039568 5955 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.5.163.188:32827: Network error: Client connection negotiation failed: client connection to 127.5.163.188:32827: connect: Connection refused (error 111)
21:13:09.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:09.107981 5955 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } attempt: 2
21:13:09.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:09.112279 5955 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.5.163.188:32827: Network error: Client connection negotiation failed: client connection to 127.5.163.188:32827: connect: Connection refused (error 111)
21:13:09.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:09.203665 5955 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } attempt: 3
21:13:09.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:09.208012 5955 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.5.163.188:32827: Network error: Client connection negotiation failed: client connection to 127.5.163.188:32827: connect: Connection refused (error 111)
21:13:09.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:09.232008 5872 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.163.189" port: 41991 } attempt: 7
21:13:09.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 11 21:13:09 dist-test-slave-2lgj krb5kdc[5786](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765487586, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.163.190@KRBTEST.COM for kudu/127.5.163.189@KRBTEST.COM
21:13:09.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:09.256407 5872 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } has no permanent_uuid. Determining permanent_uuid...
21:13:09.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:09.261520 5872 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.5.163.188:32827: Network error: Client connection negotiation failed: client connection to 127.5.163.188:32827: connect: Connection refused (error 111)
21:13:09.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:09.290196 5965 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:13:09.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:09.290699 5965 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:13:09.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:09.291312 5965 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:13:09.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:09.319972 5872 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } attempt: 1
21:13:09.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:09.323954 5872 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.5.163.188:32827: Network error: Client connection negotiation failed: client connection to 127.5.163.188:32827: connect: Connection refused (error 111)
21:13:09.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:09.325951 5965 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:13:09.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:09.326256 5965 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:13:09.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:09.326653 5965 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:13:09.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:09.326913 5965 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:13:09.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:09.327178 5965 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:13:09.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:09.327471 5965 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:13:09.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:09.327684 5965 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43381/jwks.json
21:13:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:09.351416 5955 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } attempt: 4
21:13:09.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:09.354959 5955 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.5.163.188:32827: Network error: Client connection negotiation failed: client connection to 127.5.163.188:32827: connect: Connection refused (error 111)
21:13:09.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:09.365396 5965 master_runner.cc:386] Master server non-default flags:
21:13:09.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:13:09.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.163.148:33837
21:13:09.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:13:09.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:13:09.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-2/data
21:13:09.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-2/wal
21:13:09.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:13:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:13:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.163.190:35663,127.5.163.189:41991,127.5.163.188:32827
21:13:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/kudu.keytab
21:13:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.163.188
21:13:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:13:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:13:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:13:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.163.188:32827
21:13:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:13:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:13:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:13:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43381/jwks.json
21:13:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:13:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:13:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:13:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:13:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:13:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-2/data/info.pb
21:13:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.163.188
21:13:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:13:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:13:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:13:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:13:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:13:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:13:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-2/logs
21:13:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:13:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:13:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:13:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:13:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:13:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
21:13:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:13:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Dec 2025 20:43:24 UTC on 5fd53c4cbb9d
21:13:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9469
21:13:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:13:09.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:09.366894 5965 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:13:09.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:09.367877 5965 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:13:09.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:09.386194 5975 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:13:09.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:09.405357 5872 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } attempt: 2
21:13:09.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:09.410878 5872 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.5.163.188:32827: Network error: Client connection negotiation failed: client connection to 127.5.163.188:32827: connect: Connection refused (error 111)
21:13:09.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:09.496376 5872 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } attempt: 3
21:13:09.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:09.502369 5872 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.5.163.188:32827: Network error: Client connection negotiation failed: client connection to 127.5.163.188:32827: connect: Connection refused (error 111)
21:13:09.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:09.642784 5872 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } attempt: 4
21:13:09.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:09.649490 5872 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.5.163.188:32827: Network error: Client connection negotiation failed: client connection to 127.5.163.188:32827: connect: Connection refused (error 111)
21:13:09.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:09.654305 5955 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } attempt: 5
21:13:09.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:09.660130 5955 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.5.163.188:32827: Network error: Client connection negotiation failed: client connection to 127.5.163.188:32827: connect: Connection refused (error 111)
21:13:09.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:09.932935 5872 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } attempt: 5
21:13:09.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:09.939633 5872 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.5.163.188:32827: Network error: Client connection negotiation failed: client connection to 127.5.163.188:32827: connect: Connection refused (error 111)
21:13:10.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:10.207530 5955 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } attempt: 6
21:13:10.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:10.240746 5955 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.5.163.188:32827: Network error: Client connection negotiation failed: client connection to 127.5.163.188:32827: connect: Connection refused (error 111)
21:13:10.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:10.492067 5872 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } attempt: 6
21:13:10.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:10.530761 5872 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.5.163.188:32827: Network error: Client connection negotiation failed: client connection to 127.5.163.188:32827: connect: Connection refused (error 111)
21:13:10.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:09.387251 5974 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:13:10.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:10.761694 5976 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1370 milliseconds
21:13:10.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:10.762521 5965 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.375s user 0.571s sys 0.792s
21:13:10.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:10.762863 5965 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.375s user 0.571s sys 0.792s
21:13:10.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:10.762888 5978 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:13:10.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:10.763149 5965 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:13:10.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:10.764592 5965 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:13:10.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:10.767505 5965 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:13:10.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:10.768934 5965 hybrid_clock.cc:648] HybridClock initialized: now 1765487590768829 us; error 82 us; skew 500 ppm
21:13:10.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:10.770000 5965 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:13:10.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 11 21:13:10 dist-test-slave-2lgj krb5kdc[5786](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765487590, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.163.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:13:10.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:10.777837 5965 init.cc:377] Logged in from keytab as kudu/127.5.163.188@KRBTEST.COM (short username kudu)
21:13:10.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:10.782702 5965 webserver.cc:492] Webserver started at http://127.5.163.188:34227/ using document root <none> and password file <none>
21:13:10.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:10.783562 5965 fs_manager.cc:362] Metadata directory not provided
21:13:10.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:10.783738 5965 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:13:10.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:10.784113 5965 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:13:10.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:10.788486 5965 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-2/data/instance:
21:13:10.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d65fad9630b3464aa5b089e635ea9fcb"
21:13:10.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-11 21:13:10 on dist-test-slave-2lgj"
21:13:10.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:10.789510 5965 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-2/wal/instance:
21:13:10.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d65fad9630b3464aa5b089e635ea9fcb"
21:13:10.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-11 21:13:10 on dist-test-slave-2lgj"
21:13:10.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:10.797117 5965 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.001s
21:13:10.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:10.802870 5991 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:13:10.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:10.803920 5965 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
21:13:10.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:10.804220 5965 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-2/data,/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-2/wal
21:13:10.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d65fad9630b3464aa5b089e635ea9fcb"
21:13:10.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-11 21:13:10 on dist-test-slave-2lgj"
21:13:10.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:10.804524 5965 fs_report.cc:389] FS layout report
21:13:10.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:13:10.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-2/wal
21:13:10.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-2/wal
21:13:10.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-2/data/data
21:13:10.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:13:10.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:13:10.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:13:10.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:13:10.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:13:10.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:13:10.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:13:10.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:13:10.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:13:10.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:13:10.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:13:10.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:10.897454 5965 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:13:10.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:10.904237 5965 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:13:10.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:10.904757 5965 kserver.cc:163] Server-wide thread pool size limit: 3276
21:13:10.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:10.972854 5965 rpc_server.cc:307] RPC server started. Bound to: 127.5.163.188:32827
21:13:10.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:10.972914 6043 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.5.163.188:32827 every 8 connection(s)
21:13:10.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:10.975517 5965 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-2/data/info.pb
21:13:10.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:10.981146 6044 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:13:10.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:10.984222 5774 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu as pid 5965
21:13:10.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:10.984695 5774 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-2/wal/instance
21:13:10.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:10.993989 6044 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.5.163.190" port: 35663 } has no permanent_uuid. Determining permanent_uuid...
21:13:11.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 11 21:13:11 dist-test-slave-2lgj krb5kdc[5786](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765487590, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.163.188@KRBTEST.COM for kudu/127.5.163.190@KRBTEST.COM
21:13:11.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 11 21:13:11 dist-test-slave-2lgj krb5kdc[5786](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765487584, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.5.163.188@KRBTEST.COM
21:13:11.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.024163 6044 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.5.163.189" port: 41991 } has no permanent_uuid. Determining permanent_uuid...
21:13:11.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 11 21:13:11 dist-test-slave-2lgj krb5kdc[5786](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765487590, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.163.188@KRBTEST.COM for kudu/127.5.163.189@KRBTEST.COM
21:13:11.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 11 21:13:11 dist-test-slave-2lgj krb5kdc[5786](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765487584, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.5.163.189@KRBTEST.COM
21:13:11.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.044554 6044 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } has no permanent_uuid. Determining permanent_uuid...
21:13:11.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 11 21:13:11 dist-test-slave-2lgj krb5kdc[5786](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765487590, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.163.188@KRBTEST.COM for kudu/127.5.163.188@KRBTEST.COM
21:13:11.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 11 21:13:11 dist-test-slave-2lgj krb5kdc[5786](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765487584, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.5.163.190@KRBTEST.COM
21:13:11.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.069912 6044 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb: Bootstrap starting.
21:13:11.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.076177 6044 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb: Neither blocks nor log segments found. Creating new log.
21:13:11.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.076432 5774 external_mini_cluster.cc:949] 0 TS(s) registered with all masters
21:13:11.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.077821 6044 log.cc:826] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb: Log is configured to *not* fsync() on all Append() calls
21:13:11.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.082137 6044 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb: No bootstrap required, opened a new log
21:13:11.084 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:13:11.084 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
21:13:11.085 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
21:13:11.086 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
21:13:11.087 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.5.163.190"
port: 35663
}
bound_http_address {
host: "127.5.163.190"
port: 37105
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.5.163.189"
port: 41991
}
bound_http_address {
host: "127.5.163.189"
port: 34781
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.5.163.188"
port: 32827
}
bound_http_address {
host: "127.5.163.188"
port: 34227
}
}
}
21:13:11.087 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
21:13:11.087 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:13:11.087 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
21:13:11.087 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
21:13:11.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.101537 6044 raft_consensus.cc:359] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc7d413f28fc40929b53b58b87dc209e" member_type: VOTER last_known_addr { host: "127.5.163.190" port: 35663 } } peers { permanent_uuid: "dc9b40a769d748be83c1ae82b0e2a748" member_type: VOTER last_known_addr { host: "127.5.163.189" port: 41991 } } peers { permanent_uuid: "d65fad9630b3464aa5b089e635ea9fcb" member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } }
21:13:11.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.102315 6044 raft_consensus.cc:385] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:13:11.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.102615 6044 raft_consensus.cc:740] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d65fad9630b3464aa5b089e635ea9fcb, State: Initialized, Role: FOLLOWER
21:13:11.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.103444 6044 consensus_queue.cc:260] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc7d413f28fc40929b53b58b87dc209e" member_type: VOTER last_known_addr { host: "127.5.163.190" port: 35663 } } peers { permanent_uuid: "dc9b40a769d748be83c1ae82b0e2a748" member_type: VOTER last_known_addr { host: "127.5.163.189" port: 41991 } } peers { permanent_uuid: "d65fad9630b3464aa5b089e635ea9fcb" member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } }
21:13:11.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.106261 6054 sys_catalog.cc:455] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc7d413f28fc40929b53b58b87dc209e" member_type: VOTER last_known_addr { host: "127.5.163.190" port: 35663 } } peers { permanent_uuid: "dc9b40a769d748be83c1ae82b0e2a748" member_type: VOTER last_known_addr { host: "127.5.163.189" port: 41991 } } peers { permanent_uuid: "d65fad9630b3464aa5b089e635ea9fcb" member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } } }
21:13:11.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.107147 6054 sys_catalog.cc:458] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [sys.catalog]: This master's current role is: FOLLOWER
21:13:11.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.108186 6044 sys_catalog.cc:565] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [sys.catalog]: configured and running, proceeding with master startup.
21:13:11.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 11 21:13:11 dist-test-slave-2lgj krb5kdc[5786](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765487591, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:13:11.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
21:13:11.125 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:13:11.126 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/krb5cc
21:13:11.127 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
21:13:11.128 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-user
21:13:11.128 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-user"
}
21:13:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:11.139041 6066 catalog_manager.cc:1568] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:13:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:11.139361 6066 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:13:11.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.135422 5965 master_runner.cc:186] Error getting master registration for 127.5.163.190:35663: OK, instance_id { permanent_uuid: "dc7d413f28fc40929b53b58b87dc209e" instance_seqno: 1765487586816644 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:13:11.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.157756 5965 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:13:11.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 11 21:13:11 dist-test-slave-2lgj krb5kdc[5786](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765487591, etypes {rep=17 tkt=17 ses=17}, test-user@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:13:11.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:11.160483 5998 tablet.cc:2378] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:13:11.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-user@KRBTEST.COM:
21:13:11.169 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:13:11.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.301301 5955 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } attempt: 7
21:13:11.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 11 21:13:11 dist-test-slave-2lgj krb5kdc[5786](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765487588, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.163.189@KRBTEST.COM for kudu/127.5.163.188@KRBTEST.COM
21:13:11.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.330096 5955 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dc9b40a769d748be83c1ae82b0e2a748: Bootstrap starting.
21:13:11.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.335083 5955 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P dc9b40a769d748be83c1ae82b0e2a748: Neither blocks nor log segments found. Creating new log.
21:13:11.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.336910 5955 log.cc:826] T 00000000000000000000000000000000 P dc9b40a769d748be83c1ae82b0e2a748: Log is configured to *not* fsync() on all Append() calls
21:13:11.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.338186 6054 raft_consensus.cc:493] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:13:11.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.338676 6054 raft_consensus.cc:515] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc7d413f28fc40929b53b58b87dc209e" member_type: VOTER last_known_addr { host: "127.5.163.190" port: 35663 } } peers { permanent_uuid: "dc9b40a769d748be83c1ae82b0e2a748" member_type: VOTER last_known_addr { host: "127.5.163.189" port: 41991 } } peers { permanent_uuid: "d65fad9630b3464aa5b089e635ea9fcb" member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } }
21:13:11.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.341053 6054 leader_election.cc:290] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers dc7d413f28fc40929b53b58b87dc209e (127.5.163.190:35663), dc9b40a769d748be83c1ae82b0e2a748 (127.5.163.189:41991)
21:13:11.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.341884 5955 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dc9b40a769d748be83c1ae82b0e2a748: No bootstrap required, opened a new log
21:13:11.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.368636 5955 raft_consensus.cc:359] T 00000000000000000000000000000000 P dc9b40a769d748be83c1ae82b0e2a748 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc7d413f28fc40929b53b58b87dc209e" member_type: VOTER last_known_addr { host: "127.5.163.190" port: 35663 } } peers { permanent_uuid: "dc9b40a769d748be83c1ae82b0e2a748" member_type: VOTER last_known_addr { host: "127.5.163.189" port: 41991 } } peers { permanent_uuid: "d65fad9630b3464aa5b089e635ea9fcb" member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } }
21:13:11.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.369505 5955 raft_consensus.cc:385] T 00000000000000000000000000000000 P dc9b40a769d748be83c1ae82b0e2a748 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:13:11.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.369815 5955 raft_consensus.cc:740] T 00000000000000000000000000000000 P dc9b40a769d748be83c1ae82b0e2a748 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dc9b40a769d748be83c1ae82b0e2a748, State: Initialized, Role: FOLLOWER
21:13:11.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.370688 5955 consensus_queue.cc:260] T 00000000000000000000000000000000 P dc9b40a769d748be83c1ae82b0e2a748 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc7d413f28fc40929b53b58b87dc209e" member_type: VOTER last_known_addr { host: "127.5.163.190" port: 35663 } } peers { permanent_uuid: "dc9b40a769d748be83c1ae82b0e2a748" member_type: VOTER last_known_addr { host: "127.5.163.189" port: 41991 } } peers { permanent_uuid: "d65fad9630b3464aa5b089e635ea9fcb" member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } }
21:13:11.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.375052 6072 sys_catalog.cc:455] T 00000000000000000000000000000000 P dc9b40a769d748be83c1ae82b0e2a748 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc7d413f28fc40929b53b58b87dc209e" member_type: VOTER last_known_addr { host: "127.5.163.190" port: 35663 } } peers { permanent_uuid: "dc9b40a769d748be83c1ae82b0e2a748" member_type: VOTER last_known_addr { host: "127.5.163.189" port: 41991 } } peers { permanent_uuid: "d65fad9630b3464aa5b089e635ea9fcb" member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } } }
21:13:11.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.376665 5955 sys_catalog.cc:565] T 00000000000000000000000000000000 P dc9b40a769d748be83c1ae82b0e2a748 [sys.catalog]: configured and running, proceeding with master startup.
21:13:11.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.376799 5930 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d65fad9630b3464aa5b089e635ea9fcb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dc9b40a769d748be83c1ae82b0e2a748" is_pre_election: true
21:13:11.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.377542 5930 raft_consensus.cc:2468] T 00000000000000000000000000000000 P dc9b40a769d748be83c1ae82b0e2a748 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d65fad9630b3464aa5b089e635ea9fcb in term 0.
21:13:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.376502 6072 sys_catalog.cc:458] T 00000000000000000000000000000000 P dc9b40a769d748be83c1ae82b0e2a748 [sys.catalog]: This master's current role is: FOLLOWER
21:13:11.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.382934 5994 leader_election.cc:304] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: d65fad9630b3464aa5b089e635ea9fcb, dc9b40a769d748be83c1ae82b0e2a748; no voters:
21:13:11.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.386912 6054 raft_consensus.cc:2804] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [term 0 FOLLOWER]: Leader pre-election won for term 1
21:13:11.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.388234 6054 raft_consensus.cc:493] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:13:11.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.388622 6054 raft_consensus.cc:3060] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [term 0 FOLLOWER]: Advancing to term 1
21:13:11.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.395197 6054 raft_consensus.cc:515] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc7d413f28fc40929b53b58b87dc209e" member_type: VOTER last_known_addr { host: "127.5.163.190" port: 35663 } } peers { permanent_uuid: "dc9b40a769d748be83c1ae82b0e2a748" member_type: VOTER last_known_addr { host: "127.5.163.189" port: 41991 } } peers { permanent_uuid: "d65fad9630b3464aa5b089e635ea9fcb" member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } }
21:13:11.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.400904 5930 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d65fad9630b3464aa5b089e635ea9fcb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dc9b40a769d748be83c1ae82b0e2a748"
21:13:11.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.401438 5930 raft_consensus.cc:3060] T 00000000000000000000000000000000 P dc9b40a769d748be83c1ae82b0e2a748 [term 0 FOLLOWER]: Advancing to term 1
21:13:11.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.411233 5930 raft_consensus.cc:2468] T 00000000000000000000000000000000 P dc9b40a769d748be83c1ae82b0e2a748 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d65fad9630b3464aa5b089e635ea9fcb in term 1.
21:13:11.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.414004 5994 leader_election.cc:304] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: d65fad9630b3464aa5b089e635ea9fcb, dc9b40a769d748be83c1ae82b0e2a748; no voters:
21:13:11.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.419734 6054 leader_election.cc:290] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [CANDIDATE]: Term 1 election: Requested vote from peers dc7d413f28fc40929b53b58b87dc209e (127.5.163.190:35663), dc9b40a769d748be83c1ae82b0e2a748 (127.5.163.189:41991)
21:13:11.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.420883 6054 raft_consensus.cc:2804] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [term 1 FOLLOWER]: Leader election won for term 1
21:13:11.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.422923 6054 raft_consensus.cc:697] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [term 1 LEADER]: Becoming Leader. State: Replica: d65fad9630b3464aa5b089e635ea9fcb, State: Running, Role: LEADER
21:13:11.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.424321 6054 consensus_queue.cc:237] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc7d413f28fc40929b53b58b87dc209e" member_type: VOTER last_known_addr { host: "127.5.163.190" port: 35663 } } peers { permanent_uuid: "dc9b40a769d748be83c1ae82b0e2a748" member_type: VOTER last_known_addr { host: "127.5.163.189" port: 41991 } } peers { permanent_uuid: "d65fad9630b3464aa5b089e635ea9fcb" member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } }
21:13:11.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:11.450191 6085 catalog_manager.cc:1568] T 00000000000000000000000000000000 P dc9b40a769d748be83c1ae82b0e2a748: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:13:11.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:11.450500 6085 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:13:11.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.449559 6075 sys_catalog.cc:455] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [sys.catalog]: SysCatalogTable state changed. Reason: New leader d65fad9630b3464aa5b089e635ea9fcb. Latest consensus state: current_term: 1 leader_uuid: "d65fad9630b3464aa5b089e635ea9fcb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc7d413f28fc40929b53b58b87dc209e" member_type: VOTER last_known_addr { host: "127.5.163.190" port: 35663 } } peers { permanent_uuid: "dc9b40a769d748be83c1ae82b0e2a748" member_type: VOTER last_known_addr { host: "127.5.163.189" port: 41991 } } peers { permanent_uuid: "d65fad9630b3464aa5b089e635ea9fcb" member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } } }
21:13:11.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.451587 6075 sys_catalog.cc:458] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [sys.catalog]: This master's current role is: LEADER
21:13:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.458559 6087 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
21:13:11.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.466449 6087 catalog_manager.cc:1494] Initializing Kudu cluster ID...
21:13:11.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.488658 5930 raft_consensus.cc:1275] T 00000000000000000000000000000000 P dc9b40a769d748be83c1ae82b0e2a748 [term 1 FOLLOWER]: Refusing update from remote peer d65fad9630b3464aa5b089e635ea9fcb: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:13:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.492379 6075 consensus_queue.cc:1048] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc9b40a769d748be83c1ae82b0e2a748" member_type: VOTER last_known_addr { host: "127.5.163.189" port: 41991 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:13:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.440845 5881 master_runner.cc:186] Error getting master registration for 127.5.163.190:35663: OK, instance_id { permanent_uuid: "dc7d413f28fc40929b53b58b87dc209e" instance_seqno: 1765487586816644 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:13:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.512456 5881 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:13:11.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.564193 5872 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } attempt: 7
21:13:11.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.564193 6088 mvcc.cc:204] Tried to move back new op lower bound from 7231437174717726720 to 7231437174496894976. Current Snapshot: MvccSnapshot[applied={T|T < 7231437174717726720}]
21:13:11.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.566099 6072 sys_catalog.cc:455] T 00000000000000000000000000000000 P dc9b40a769d748be83c1ae82b0e2a748 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d65fad9630b3464aa5b089e635ea9fcb. Latest consensus state: current_term: 1 leader_uuid: "d65fad9630b3464aa5b089e635ea9fcb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc7d413f28fc40929b53b58b87dc209e" member_type: VOTER last_known_addr { host: "127.5.163.190" port: 35663 } } peers { permanent_uuid: "dc9b40a769d748be83c1ae82b0e2a748" member_type: VOTER last_known_addr { host: "127.5.163.189" port: 41991 } } peers { permanent_uuid: "d65fad9630b3464aa5b089e635ea9fcb" member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } } }
21:13:11.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.567926 6072 sys_catalog.cc:458] T 00000000000000000000000000000000 P dc9b40a769d748be83c1ae82b0e2a748 [sys.catalog]: This master's current role is: FOLLOWER
21:13:11.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.570693 6054 sys_catalog.cc:455] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "d65fad9630b3464aa5b089e635ea9fcb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc7d413f28fc40929b53b58b87dc209e" member_type: VOTER last_known_addr { host: "127.5.163.190" port: 35663 } } peers { permanent_uuid: "dc9b40a769d748be83c1ae82b0e2a748" member_type: VOTER last_known_addr { host: "127.5.163.189" port: 41991 } } peers { permanent_uuid: "d65fad9630b3464aa5b089e635ea9fcb" member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } } }
21:13:11.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.571373 6054 sys_catalog.cc:458] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [sys.catalog]: This master's current role is: LEADER
21:13:11.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.575016 6087 catalog_manager.cc:1357] Generated new cluster ID: 598f0d2c2b304c9784e7cb98a66bf96e
21:13:11.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.575275 6087 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
21:13:11.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.626673 6072 sys_catalog.cc:455] T 00000000000000000000000000000000 P dc9b40a769d748be83c1ae82b0e2a748 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "d65fad9630b3464aa5b089e635ea9fcb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc7d413f28fc40929b53b58b87dc209e" member_type: VOTER last_known_addr { host: "127.5.163.190" port: 35663 } } peers { permanent_uuid: "dc9b40a769d748be83c1ae82b0e2a748" member_type: VOTER last_known_addr { host: "127.5.163.189" port: 41991 } } peers { permanent_uuid: "d65fad9630b3464aa5b089e635ea9fcb" member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } } }
21:13:11.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.627434 6072 sys_catalog.cc:458] T 00000000000000000000000000000000 P dc9b40a769d748be83c1ae82b0e2a748 [sys.catalog]: This master's current role is: FOLLOWER
21:13:11.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 11 21:13:11 dist-test-slave-2lgj krb5kdc[5786](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765487586, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.163.190@KRBTEST.COM for kudu/127.5.163.188@KRBTEST.COM
21:13:11.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.672447 5872 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dc7d413f28fc40929b53b58b87dc209e: Bootstrap starting.
21:13:11.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.673611 6087 catalog_manager.cc:1380] Generated new certificate authority record
21:13:11.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.675770 6087 catalog_manager.cc:1514] Loading token signing keys...
21:13:11.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.684789 5872 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P dc7d413f28fc40929b53b58b87dc209e: Neither blocks nor log segments found. Creating new log.
21:13:11.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.699887 5872 log.cc:826] T 00000000000000000000000000000000 P dc7d413f28fc40929b53b58b87dc209e: Log is configured to *not* fsync() on all Append() calls
21:13:11.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.707811 5872 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dc7d413f28fc40929b53b58b87dc209e: No bootstrap required, opened a new log
21:13:11.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.727897 6087 catalog_manager.cc:6027] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb: Generated new TSK 0
21:13:11.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.728950 6087 catalog_manager.cc:1524] Initializing in-progress tserver states...
21:13:11.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.742383 5872 raft_consensus.cc:359] T 00000000000000000000000000000000 P dc7d413f28fc40929b53b58b87dc209e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc7d413f28fc40929b53b58b87dc209e" member_type: VOTER last_known_addr { host: "127.5.163.190" port: 35663 } } peers { permanent_uuid: "dc9b40a769d748be83c1ae82b0e2a748" member_type: VOTER last_known_addr { host: "127.5.163.189" port: 41991 } } peers { permanent_uuid: "d65fad9630b3464aa5b089e635ea9fcb" member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } }
21:13:11.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.743181 5872 raft_consensus.cc:385] T 00000000000000000000000000000000 P dc7d413f28fc40929b53b58b87dc209e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:13:11.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.743575 5872 raft_consensus.cc:740] T 00000000000000000000000000000000 P dc7d413f28fc40929b53b58b87dc209e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dc7d413f28fc40929b53b58b87dc209e, State: Initialized, Role: FOLLOWER
21:13:11.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.746225 5872 consensus_queue.cc:260] T 00000000000000000000000000000000 P dc7d413f28fc40929b53b58b87dc209e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc7d413f28fc40929b53b58b87dc209e" member_type: VOTER last_known_addr { host: "127.5.163.190" port: 35663 } } peers { permanent_uuid: "dc9b40a769d748be83c1ae82b0e2a748" member_type: VOTER last_known_addr { host: "127.5.163.189" port: 41991 } } peers { permanent_uuid: "d65fad9630b3464aa5b089e635ea9fcb" member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } }
21:13:11.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.750700 5872 sys_catalog.cc:565] T 00000000000000000000000000000000 P dc7d413f28fc40929b53b58b87dc209e [sys.catalog]: configured and running, proceeding with master startup.
21:13:11.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.750851 5846 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d65fad9630b3464aa5b089e635ea9fcb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dc7d413f28fc40929b53b58b87dc209e"
21:13:11.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.751441 5846 raft_consensus.cc:3060] T 00000000000000000000000000000000 P dc7d413f28fc40929b53b58b87dc209e [term 0 FOLLOWER]: Advancing to term 1
21:13:11.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.749462 6096 sys_catalog.cc:455] T 00000000000000000000000000000000 P dc7d413f28fc40929b53b58b87dc209e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc7d413f28fc40929b53b58b87dc209e" member_type: VOTER last_known_addr { host: "127.5.163.190" port: 35663 } } peers { permanent_uuid: "dc9b40a769d748be83c1ae82b0e2a748" member_type: VOTER last_known_addr { host: "127.5.163.189" port: 41991 } } peers { permanent_uuid: "d65fad9630b3464aa5b089e635ea9fcb" member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } } }
21:13:11.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.753459 6096 sys_catalog.cc:458] T 00000000000000000000000000000000 P dc7d413f28fc40929b53b58b87dc209e [sys.catalog]: This master's current role is: FOLLOWER
21:13:11.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.755504 5847 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d65fad9630b3464aa5b089e635ea9fcb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dc7d413f28fc40929b53b58b87dc209e" is_pre_election: true
21:13:11.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.762966 5846 raft_consensus.cc:2468] T 00000000000000000000000000000000 P dc7d413f28fc40929b53b58b87dc209e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d65fad9630b3464aa5b089e635ea9fcb in term 1.
21:13:11.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.764289 5845 raft_consensus.cc:1275] T 00000000000000000000000000000000 P dc7d413f28fc40929b53b58b87dc209e [term 1 FOLLOWER]: Refusing update from remote peer d65fad9630b3464aa5b089e635ea9fcb: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:13:11.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.769520 6054 consensus_queue.cc:1048] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc7d413f28fc40929b53b58b87dc209e" member_type: VOTER last_known_addr { host: "127.5.163.190" port: 35663 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:13:11.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:11.777364 5826 tablet.cc:2378] T 00000000000000000000000000000000 P dc7d413f28fc40929b53b58b87dc209e: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:13:11.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.844372 6096 sys_catalog.cc:455] T 00000000000000000000000000000000 P dc7d413f28fc40929b53b58b87dc209e [sys.catalog]: SysCatalogTable state changed. Reason: New leader d65fad9630b3464aa5b089e635ea9fcb. Latest consensus state: current_term: 1 leader_uuid: "d65fad9630b3464aa5b089e635ea9fcb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc7d413f28fc40929b53b58b87dc209e" member_type: VOTER last_known_addr { host: "127.5.163.190" port: 35663 } } peers { permanent_uuid: "dc9b40a769d748be83c1ae82b0e2a748" member_type: VOTER last_known_addr { host: "127.5.163.189" port: 41991 } } peers { permanent_uuid: "d65fad9630b3464aa5b089e635ea9fcb" member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } } }
21:13:11.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.845016 6096 sys_catalog.cc:458] T 00000000000000000000000000000000 P dc7d413f28fc40929b53b58b87dc209e [sys.catalog]: This master's current role is: FOLLOWER
21:13:11.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.846380 6102 sys_catalog.cc:455] T 00000000000000000000000000000000 P dc7d413f28fc40929b53b58b87dc209e [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "d65fad9630b3464aa5b089e635ea9fcb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc7d413f28fc40929b53b58b87dc209e" member_type: VOTER last_known_addr { host: "127.5.163.190" port: 35663 } } peers { permanent_uuid: "dc9b40a769d748be83c1ae82b0e2a748" member_type: VOTER last_known_addr { host: "127.5.163.189" port: 41991 } } peers { permanent_uuid: "d65fad9630b3464aa5b089e635ea9fcb" member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } } }
21:13:11.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.847051 6102 sys_catalog.cc:458] T 00000000000000000000000000000000 P dc7d413f28fc40929b53b58b87dc209e [sys.catalog]: This master's current role is: FOLLOWER
21:13:11.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:11.860041 6112 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
21:13:12.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:12.363831 6054 sys_catalog.cc:455] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "d65fad9630b3464aa5b089e635ea9fcb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc7d413f28fc40929b53b58b87dc209e" member_type: VOTER last_known_addr { host: "127.5.163.190" port: 35663 } } peers { permanent_uuid: "dc9b40a769d748be83c1ae82b0e2a748" member_type: VOTER last_known_addr { host: "127.5.163.189" port: 41991 } } peers { permanent_uuid: "d65fad9630b3464aa5b089e635ea9fcb" member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } } }
21:13:12.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:12.367122 6054 sys_catalog.cc:458] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [sys.catalog]: This master's current role is: LEADER
21:13:12.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:12.455935 6085 catalog_manager.cc:1269] Loaded cluster ID: 598f0d2c2b304c9784e7cb98a66bf96e
21:13:12.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:12.456228 6085 catalog_manager.cc:1562] T 00000000000000000000000000000000 P dc9b40a769d748be83c1ae82b0e2a748: loading cluster ID for follower catalog manager: success
21:13:12.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:12.463083 6085 catalog_manager.cc:1584] T 00000000000000000000000000000000 P dc9b40a769d748be83c1ae82b0e2a748: acquiring CA information for follower catalog manager: success
21:13:12.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:12.468108 6085 catalog_manager.cc:1612] T 00000000000000000000000000000000 P dc9b40a769d748be83c1ae82b0e2a748: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:13:12.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:12.878712 6112 catalog_manager.cc:1269] Loaded cluster ID: 598f0d2c2b304c9784e7cb98a66bf96e
21:13:12.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:12.878983 6112 catalog_manager.cc:1562] T 00000000000000000000000000000000 P dc7d413f28fc40929b53b58b87dc209e: loading cluster ID for follower catalog manager: success
21:13:12.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:12.891070 6112 catalog_manager.cc:1584] T 00000000000000000000000000000000 P dc7d413f28fc40929b53b58b87dc209e: acquiring CA information for follower catalog manager: success
21:13:12.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:12.897094 6112 catalog_manager.cc:1612] T 00000000000000000000000000000000 P dc7d413f28fc40929b53b58b87dc209e: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:13:14.171 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.5.163.188:32827
21:13:14.171 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
21:13:14.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:14.172554 5774 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu with pid 5965
21:13:14.207 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:13:14.207 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.5.163.190:35663
21:13:14.207 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
21:13:14.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:14.207922 5774 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu with pid 5802
21:13:14.232 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:13:14.232 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.5.163.189:41991
21:13:14.232 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
21:13:14.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:14.232535 5774 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu with pid 5881
21:13:14.257 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:13:14.257 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.5.163.188:32827
21:13:14.257 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
21:13:14.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:14.259250 5774 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu
21:13:14.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu
21:13:14.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-2/wal
21:13:14.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-2/data
21:13:14.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:13:14.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:13:14.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:13:14.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:13:14.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:13:14.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:13:14.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-2/logs
21:13:14.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-2/data/info.pb
21:13:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:13:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:13:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:13:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:13:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:13:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:13:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:13:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:13:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:13:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:13:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:13:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:13:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.163.188:32827
21:13:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.163.188
21:13:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34227
21:13:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/kudu.keytab
21:13:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.163.188
21:13:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:13:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:13:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:13:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:13:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.163.148:33837
21:13:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:13:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:13:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:13:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:13:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.163.190:35663,127.5.163.189:41991,127.5.163.188:32827
21:13:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:13:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43381/jwks.json
21:13:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:13:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:13:14.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:14.564963 6120 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:13:14.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:14.565466 6120 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:13:14.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:14.566095 6120 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:13:14.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:14.596836 6120 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:13:14.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:14.597049 6120 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:13:14.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:14.597306 6120 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:13:14.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:14.597497 6120 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:13:14.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:14.597672 6120 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:13:14.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:14.597961 6120 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:13:14.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:14.598170 6120 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43381/jwks.json
21:13:14.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:14.634531 6120 master_runner.cc:386] Master server non-default flags:
21:13:14.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:13:14.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.163.148:33837
21:13:14.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:13:14.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:13:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-2/data
21:13:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-2/wal
21:13:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:13:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:13:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.163.190:35663,127.5.163.189:41991,127.5.163.188:32827
21:13:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/kudu.keytab
21:13:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.163.188
21:13:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:13:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:13:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:13:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.163.188:32827
21:13:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:13:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:13:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:13:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43381/jwks.json
21:13:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:13:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:13:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:13:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:13:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:13:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-2/data/info.pb
21:13:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.163.188
21:13:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34227
21:13:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:13:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:13:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:13:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:13:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:13:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-2/logs
21:13:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:13:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:13:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:13:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:13:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:13:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
21:13:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:13:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Dec 2025 20:43:24 UTC on 5fd53c4cbb9d
21:13:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9469
21:13:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:13:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:14.635687 6120 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:13:14.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:14.636677 6120 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:13:14.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:14.646816 6126 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:13:15.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:14.647393 6127 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:13:15.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:15.932271 6129 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:13:15.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:15.934005 6128 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1283 milliseconds
21:13:15.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:15.935279 6120 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.288s user 0.485s sys 0.780s
21:13:15.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:15.935611 6120 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.289s user 0.485s sys 0.780s
21:13:15.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:15.935896 6120 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:13:15.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:15.937350 6120 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:13:15.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:15.940357 6120 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:13:15.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:15.941799 6120 hybrid_clock.cc:648] HybridClock initialized: now 1765487595941715 us; error 58 us; skew 500 ppm
21:13:15.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:15.942875 6120 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:13:15.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 11 21:13:15 dist-test-slave-2lgj krb5kdc[5786](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765487595, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.163.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:13:15.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:15.950794 6120 init.cc:377] Logged in from keytab as kudu/127.5.163.188@KRBTEST.COM (short username kudu)
21:13:15.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:15.956116 6120 webserver.cc:492] Webserver started at http://127.5.163.188:34227/ using document root <none> and password file <none>
21:13:15.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:15.956986 6120 fs_manager.cc:362] Metadata directory not provided
21:13:15.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:15.957190 6120 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:13:15.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:15.966058 6120 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.007s sys 0.000s
21:13:15.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:15.970678 6140 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:13:15.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:15.971850 6120 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
21:13:15.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:15.972131 6120 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-2/data,/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-2/wal
21:13:15.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d65fad9630b3464aa5b089e635ea9fcb"
21:13:15.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-11 21:13:10 on dist-test-slave-2lgj"
21:13:15.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:15.974076 6120 fs_report.cc:389] FS layout report
21:13:15.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:13:15.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-2/wal
21:13:15.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-2/wal
21:13:15.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-2/data/data
21:13:15.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:13:15.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:13:15.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:13:15.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:13:15.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:13:15.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:13:15.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:13:15.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:13:15.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:13:15.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:13:15.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:13:16.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:16.038498 6120 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:13:16.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:16.043623 6120 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:13:16.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:16.044000 6120 kserver.cc:163] Server-wide thread pool size limit: 3276
21:13:16.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:16.112558 6120 rpc_server.cc:307] RPC server started. Bound to: 127.5.163.188:32827
21:13:16.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:16.112677 6191 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.5.163.188:32827 every 8 connection(s)
21:13:16.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:16.115366 6120 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-2/data/info.pb
21:13:16.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:16.118654 5774 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu as pid 6120
21:13:16.119 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:13:16.119 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.5.163.190:35663
21:13:16.119 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
21:13:16.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:16.120597 5774 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu
21:13:16.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu
21:13:16.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-0/wal
21:13:16.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-0/data
21:13:16.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:13:16.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:13:16.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:13:16.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:13:16.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:13:16.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:13:16.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-0/logs
21:13:16.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-0/data/info.pb
21:13:16.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:13:16.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:13:16.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:13:16.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:13:16.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:13:16.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.163.190:35663
21:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.163.190
21:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37105
21:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/kudu.keytab
21:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.163.190
21:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.163.148:33837
21:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.163.190:35663,127.5.163.189:41991,127.5.163.188:32827
21:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43381/jwks.json
21:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:13:16.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:16.126716 6192 sys_catalog.cc:263] Verifying existing consensus state
21:13:16.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:16.131971 6192 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb: Bootstrap starting.
21:13:16.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:16.168076 6192 log.cc:826] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb: Log is configured to *not* fsync() on all Append() calls
21:13:16.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:16.182843 6192 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:13:16.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:16.183696 6192 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb: Bootstrap complete.
21:13:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:16.215831 6192 raft_consensus.cc:359] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc7d413f28fc40929b53b58b87dc209e" member_type: VOTER last_known_addr { host: "127.5.163.190" port: 35663 } } peers { permanent_uuid: "dc9b40a769d748be83c1ae82b0e2a748" member_type: VOTER last_known_addr { host: "127.5.163.189" port: 41991 } } peers { permanent_uuid: "d65fad9630b3464aa5b089e635ea9fcb" member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } }
21:13:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:16.218946 6192 raft_consensus.cc:740] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: d65fad9630b3464aa5b089e635ea9fcb, State: Initialized, Role: FOLLOWER
21:13:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:16.219867 6192 consensus_queue.cc:260] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc7d413f28fc40929b53b58b87dc209e" member_type: VOTER last_known_addr { host: "127.5.163.190" port: 35663 } } peers { permanent_uuid: "dc9b40a769d748be83c1ae82b0e2a748" member_type: VOTER last_known_addr { host: "127.5.163.189" port: 41991 } } peers { permanent_uuid: "d65fad9630b3464aa5b089e635ea9fcb" member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } }
21:13:16.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:16.226508 6196 sys_catalog.cc:455] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc7d413f28fc40929b53b58b87dc209e" member_type: VOTER last_known_addr { host: "127.5.163.190" port: 35663 } } peers { permanent_uuid: "dc9b40a769d748be83c1ae82b0e2a748" member_type: VOTER last_known_addr { host: "127.5.163.189" port: 41991 } } peers { permanent_uuid: "d65fad9630b3464aa5b089e635ea9fcb" member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } } }
21:13:16.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:16.227294 6196 sys_catalog.cc:458] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [sys.catalog]: This master's current role is: FOLLOWER
21:13:16.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:16.228271 6192 sys_catalog.cc:565] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [sys.catalog]: configured and running, proceeding with master startup.
21:13:16.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:16.256610 6207 catalog_manager.cc:1269] Loaded cluster ID: 598f0d2c2b304c9784e7cb98a66bf96e
21:13:16.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:16.257066 6207 catalog_manager.cc:1562] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb: loading cluster ID for follower catalog manager: success
21:13:16.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:16.265133 6207 catalog_manager.cc:1584] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb: acquiring CA information for follower catalog manager: success
21:13:16.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:16.270700 6207 catalog_manager.cc:1612] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:13:16.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:16.253445 6120 master_runner.cc:186] Error getting master registration for 127.5.163.190:35663: Network error: Client connection negotiation failed: client connection to 127.5.163.190:35663: connect: Connection refused (error 111),
21:13:16.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:16.281152 6120 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:13:16.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:16.440389 6194 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:13:16.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:16.440898 6194 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:13:16.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:16.441543 6194 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:13:16.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:16.472520 6194 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:13:16.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:16.472735 6194 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:13:16.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:16.473039 6194 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:13:16.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:16.473284 6194 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:13:16.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:16.473531 6194 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:13:16.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:16.473742 6194 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:13:16.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:16.473933 6194 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43381/jwks.json
21:13:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:16.509990 6194 master_runner.cc:386] Master server non-default flags:
21:13:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:13:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.163.148:33837
21:13:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:13:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:13:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-0/data
21:13:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-0/wal
21:13:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:13:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:13:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.163.190:35663,127.5.163.189:41991,127.5.163.188:32827
21:13:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/kudu.keytab
21:13:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.163.190
21:13:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:13:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:13:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:13:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.163.190:35663
21:13:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:13:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:13:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:13:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43381/jwks.json
21:13:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:13:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:13:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:13:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:13:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:13:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-0/data/info.pb
21:13:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.163.190
21:13:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37105
21:13:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:13:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:13:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:13:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:13:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:13:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-0/logs
21:13:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:13:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:13:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:13:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:13:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:13:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
21:13:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:13:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Dec 2025 20:43:24 UTC on 5fd53c4cbb9d
21:13:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9469
21:13:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:13:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:16.511255 6194 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:13:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:16.512419 6194 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:13:16.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:16.526087 6215 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:13:17.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:17.788825 6222 raft_consensus.cc:493] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:13:17.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:17.789448 6222 raft_consensus.cc:515] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc7d413f28fc40929b53b58b87dc209e" member_type: VOTER last_known_addr { host: "127.5.163.190" port: 35663 } } peers { permanent_uuid: "dc9b40a769d748be83c1ae82b0e2a748" member_type: VOTER last_known_addr { host: "127.5.163.189" port: 41991 } } peers { permanent_uuid: "d65fad9630b3464aa5b089e635ea9fcb" member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } }
21:13:17.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:17.811895 6141 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.5.163.190:35663: connect: Connection refused (error 111)
21:13:17.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:17.817986 6222 leader_election.cc:290] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers dc7d413f28fc40929b53b58b87dc209e (127.5.163.190:35663), dc9b40a769d748be83c1ae82b0e2a748 (127.5.163.189:41991)
21:13:17.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:17.827548 6142 leader_election.cc:336] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer dc9b40a769d748be83c1ae82b0e2a748 (127.5.163.189:41991): Network error: Client connection negotiation failed: client connection to 127.5.163.189:41991: connect: Connection refused (error 111)
21:13:17.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:17.837512 6141 leader_election.cc:336] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer dc7d413f28fc40929b53b58b87dc209e (127.5.163.190:35663): Network error: Client connection negotiation failed: client connection to 127.5.163.190:35663: connect: Connection refused (error 111)
21:13:17.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:17.838023 6141 leader_election.cc:304] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: d65fad9630b3464aa5b089e635ea9fcb; no voters: dc7d413f28fc40929b53b58b87dc209e, dc9b40a769d748be83c1ae82b0e2a748
21:13:17.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:17.838860 6222 raft_consensus.cc:2749] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
21:13:18.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:17.928653 6214 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 6194
21:13:18.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:18.282820 6214 kernel_stack_watchdog.cc:198] Thread 6194 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 398ms:
21:13:18.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
21:13:18.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
21:13:18.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
21:13:18.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
21:13:18.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:18.283711 6194 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.756s user 0.579s sys 1.012s
21:13:18.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:18.284082 6194 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.756s user 0.579s sys 1.012s
21:13:18.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:16.529316 6216 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:13:18.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:18.286229 6218 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:13:18.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:18.290060 6217 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1761 milliseconds
21:13:18.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:18.290139 6194 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:13:18.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:18.291610 6194 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:13:18.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:18.294551 6194 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:13:18.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:18.295974 6194 hybrid_clock.cc:648] HybridClock initialized: now 1765487598295886 us; error 79 us; skew 500 ppm
21:13:18.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:18.297065 6194 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:13:18.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 11 21:13:18 dist-test-slave-2lgj krb5kdc[5786](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765487598, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.163.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:13:18.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:18.304365 6194 init.cc:377] Logged in from keytab as kudu/127.5.163.190@KRBTEST.COM (short username kudu)
21:13:18.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:18.309919 6194 webserver.cc:492] Webserver started at http://127.5.163.190:37105/ using document root <none> and password file <none>
21:13:18.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:18.311091 6194 fs_manager.cc:362] Metadata directory not provided
21:13:18.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:18.311365 6194 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:13:18.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:18.321476 6194 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.006s sys 0.000s
21:13:18.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:18.327199 6231 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:13:18.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:18.328505 6194 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
21:13:18.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:18.328908 6194 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-0/data,/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-0/wal
21:13:18.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dc7d413f28fc40929b53b58b87dc209e"
21:13:18.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-11 21:13:06 on dist-test-slave-2lgj"
21:13:18.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:18.331660 6194 fs_report.cc:389] FS layout report
21:13:18.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:13:18.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-0/wal
21:13:18.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-0/wal
21:13:18.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-0/data/data
21:13:18.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:13:18.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:13:18.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:13:18.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:13:18.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:13:18.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:13:18.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:13:18.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:13:18.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:13:18.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:13:18.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:13:18.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:18.412686 6194 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:13:18.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:18.421538 6194 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:13:18.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:18.422034 6194 kserver.cc:163] Server-wide thread pool size limit: 3276
21:13:18.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:18.501266 6194 rpc_server.cc:307] RPC server started. Bound to: 127.5.163.190:35663
21:13:18.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:18.501384 6283 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.5.163.190:35663 every 8 connection(s)
21:13:18.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:18.503974 6194 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-0/data/info.pb
21:13:18.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:18.512869 5774 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu as pid 6194
21:13:18.513 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:13:18.513 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.5.163.189:41991
21:13:18.513 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
21:13:18.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:18.515336 5774 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu
21:13:18.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu
21:13:18.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-1/wal
21:13:18.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-1/data
21:13:18.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:13:18.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:13:18.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:13:18.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:13:18.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:13:18.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:13:18.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-1/logs
21:13:18.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-1/data/info.pb
21:13:18.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:13:18.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:13:18.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:13:18.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:13:18.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:13:18.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:13:18.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:13:18.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:13:18.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:13:18.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:13:18.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:13:18.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:13:18.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.163.189:41991
21:13:18.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.163.189
21:13:18.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34781
21:13:18.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/kudu.keytab
21:13:18.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.163.189
21:13:18.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:13:18.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:13:18.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:13:18.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:13:18.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.163.148:33837
21:13:18.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:13:18.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:13:18.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:13:18.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:13:18.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.163.190:35663,127.5.163.189:41991,127.5.163.188:32827
21:13:18.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:13:18.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43381/jwks.json
21:13:18.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:13:18.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:13:18.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:18.516482 6284 sys_catalog.cc:263] Verifying existing consensus state
21:13:18.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:18.524027 6284 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dc7d413f28fc40929b53b58b87dc209e: Bootstrap starting.
21:13:18.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:18.574162 6284 log.cc:826] T 00000000000000000000000000000000 P dc7d413f28fc40929b53b58b87dc209e: Log is configured to *not* fsync() on all Append() calls
21:13:18.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:18.588466 6284 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dc7d413f28fc40929b53b58b87dc209e: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:13:18.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:18.589222 6284 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dc7d413f28fc40929b53b58b87dc209e: Bootstrap complete.
21:13:18.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:18.611325 6284 raft_consensus.cc:359] T 00000000000000000000000000000000 P dc7d413f28fc40929b53b58b87dc209e [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc7d413f28fc40929b53b58b87dc209e" member_type: VOTER last_known_addr { host: "127.5.163.190" port: 35663 } } peers { permanent_uuid: "dc9b40a769d748be83c1ae82b0e2a748" member_type: VOTER last_known_addr { host: "127.5.163.189" port: 41991 } } peers { permanent_uuid: "d65fad9630b3464aa5b089e635ea9fcb" member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } }
21:13:18.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:18.613616 6284 raft_consensus.cc:740] T 00000000000000000000000000000000 P dc7d413f28fc40929b53b58b87dc209e [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: dc7d413f28fc40929b53b58b87dc209e, State: Initialized, Role: FOLLOWER
21:13:18.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:18.614496 6284 consensus_queue.cc:260] T 00000000000000000000000000000000 P dc7d413f28fc40929b53b58b87dc209e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc7d413f28fc40929b53b58b87dc209e" member_type: VOTER last_known_addr { host: "127.5.163.190" port: 35663 } } peers { permanent_uuid: "dc9b40a769d748be83c1ae82b0e2a748" member_type: VOTER last_known_addr { host: "127.5.163.189" port: 41991 } } peers { permanent_uuid: "d65fad9630b3464aa5b089e635ea9fcb" member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } }
21:13:18.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:18.617112 6288 sys_catalog.cc:455] T 00000000000000000000000000000000 P dc7d413f28fc40929b53b58b87dc209e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc7d413f28fc40929b53b58b87dc209e" member_type: VOTER last_known_addr { host: "127.5.163.190" port: 35663 } } peers { permanent_uuid: "dc9b40a769d748be83c1ae82b0e2a748" member_type: VOTER last_known_addr { host: "127.5.163.189" port: 41991 } } peers { permanent_uuid: "d65fad9630b3464aa5b089e635ea9fcb" member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } } }
21:13:18.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:18.618001 6288 sys_catalog.cc:458] T 00000000000000000000000000000000 P dc7d413f28fc40929b53b58b87dc209e [sys.catalog]: This master's current role is: FOLLOWER
21:13:18.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:18.618934 6284 sys_catalog.cc:565] T 00000000000000000000000000000000 P dc7d413f28fc40929b53b58b87dc209e [sys.catalog]: configured and running, proceeding with master startup.
21:13:18.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:18.660796 6299 catalog_manager.cc:1269] Loaded cluster ID: 598f0d2c2b304c9784e7cb98a66bf96e
21:13:18.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:18.661053 6299 catalog_manager.cc:1562] T 00000000000000000000000000000000 P dc7d413f28fc40929b53b58b87dc209e: loading cluster ID for follower catalog manager: success
21:13:18.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:18.668344 6299 catalog_manager.cc:1584] T 00000000000000000000000000000000 P dc7d413f28fc40929b53b58b87dc209e: acquiring CA information for follower catalog manager: success
21:13:18.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:18.673741 6299 catalog_manager.cc:1612] T 00000000000000000000000000000000 P dc7d413f28fc40929b53b58b87dc209e: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:13:18.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 11 21:13:18 dist-test-slave-2lgj krb5kdc[5786](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765487598, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.163.190@KRBTEST.COM for kudu/127.5.163.190@KRBTEST.COM
21:13:18.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:18.707000 6194 master_runner.cc:186] Error getting master registration for 127.5.163.189:41991: Network error: Client connection negotiation failed: client connection to 127.5.163.189:41991: connect: Connection refused (error 111),
21:13:18.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:18.735641 6194 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:13:18.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:18.978600 6286 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:13:18.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:18.979338 6286 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:13:18.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:18.980401 6286 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:13:19.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:19.037516 6286 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:13:19.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:19.037853 6286 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:13:19.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:19.038323 6286 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:13:19.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:19.038652 6286 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:13:19.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:19.038970 6286 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:13:19.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:19.039291 6286 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:13:19.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:19.039527 6286 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43381/jwks.json
21:13:19.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:19.104118 6286 master_runner.cc:386] Master server non-default flags:
21:13:19.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:13:19.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.163.148:33837
21:13:19.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:13:19.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:13:19.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-1/data
21:13:19.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-1/wal
21:13:19.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:13:19.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:13:19.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.163.190:35663,127.5.163.189:41991,127.5.163.188:32827
21:13:19.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/kudu.keytab
21:13:19.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.163.189
21:13:19.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:13:19.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:13:19.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:13:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.163.189:41991
21:13:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:13:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:13:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:13:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43381/jwks.json
21:13:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:13:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:13:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:13:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:13:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:13:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-1/data/info.pb
21:13:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.163.189
21:13:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34781
21:13:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:13:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:13:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:13:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:13:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:13:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-1/logs
21:13:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:13:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:13:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:13:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:13:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:13:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
21:13:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:13:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Dec 2025 20:43:24 UTC on 5fd53c4cbb9d
21:13:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9469
21:13:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:13:19.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:19.105724 6286 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:13:19.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:19.107080 6286 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:13:19.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:19.142527 6310 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:13:19.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:19.570434 6319 raft_consensus.cc:493] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:13:19.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:19.573489 6319 raft_consensus.cc:515] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc7d413f28fc40929b53b58b87dc209e" member_type: VOTER last_known_addr { host: "127.5.163.190" port: 35663 } } peers { permanent_uuid: "dc9b40a769d748be83c1ae82b0e2a748" member_type: VOTER last_known_addr { host: "127.5.163.189" port: 41991 } } peers { permanent_uuid: "d65fad9630b3464aa5b089e635ea9fcb" member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } }
21:13:19.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:19.612325 6319 leader_election.cc:290] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers dc7d413f28fc40929b53b58b87dc209e (127.5.163.190:35663), dc9b40a769d748be83c1ae82b0e2a748 (127.5.163.189:41991)
21:13:19.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:19.634475 6259 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d65fad9630b3464aa5b089e635ea9fcb" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "dc7d413f28fc40929b53b58b87dc209e" is_pre_election: true
21:13:19.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:19.635366 6259 raft_consensus.cc:2468] T 00000000000000000000000000000000 P dc7d413f28fc40929b53b58b87dc209e [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d65fad9630b3464aa5b089e635ea9fcb in term 1.
21:13:19.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:19.635396 6142 leader_election.cc:336] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer dc9b40a769d748be83c1ae82b0e2a748 (127.5.163.189:41991): Network error: Client connection negotiation failed: client connection to 127.5.163.189:41991: connect: Connection refused (error 111)
21:13:19.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:19.636647 6141 leader_election.cc:304] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: d65fad9630b3464aa5b089e635ea9fcb, dc7d413f28fc40929b53b58b87dc209e; no voters: dc9b40a769d748be83c1ae82b0e2a748
21:13:19.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:19.637365 6319 raft_consensus.cc:2804] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [term 1 FOLLOWER]: Leader pre-election won for term 2
21:13:19.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:19.637910 6319 raft_consensus.cc:493] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:13:19.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:19.638474 6319 raft_consensus.cc:3060] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [term 1 FOLLOWER]: Advancing to term 2
21:13:19.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:19.647691 6319 raft_consensus.cc:515] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc7d413f28fc40929b53b58b87dc209e" member_type: VOTER last_known_addr { host: "127.5.163.190" port: 35663 } } peers { permanent_uuid: "dc9b40a769d748be83c1ae82b0e2a748" member_type: VOTER last_known_addr { host: "127.5.163.189" port: 41991 } } peers { permanent_uuid: "d65fad9630b3464aa5b089e635ea9fcb" member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } }
21:13:19.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:19.663707 6259 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d65fad9630b3464aa5b089e635ea9fcb" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "dc7d413f28fc40929b53b58b87dc209e"
21:13:19.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:19.664254 6259 raft_consensus.cc:3060] T 00000000000000000000000000000000 P dc7d413f28fc40929b53b58b87dc209e [term 1 FOLLOWER]: Advancing to term 2
21:13:19.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:19.668984 6142 leader_election.cc:336] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer dc9b40a769d748be83c1ae82b0e2a748 (127.5.163.189:41991): Network error: Client connection negotiation failed: client connection to 127.5.163.189:41991: connect: Connection refused (error 111)
21:13:19.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:19.669871 6319 leader_election.cc:290] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [CANDIDATE]: Term 2 election: Requested vote from peers dc7d413f28fc40929b53b58b87dc209e (127.5.163.190:35663), dc9b40a769d748be83c1ae82b0e2a748 (127.5.163.189:41991)
21:13:19.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:19.673056 6259 raft_consensus.cc:2468] T 00000000000000000000000000000000 P dc7d413f28fc40929b53b58b87dc209e [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d65fad9630b3464aa5b089e635ea9fcb in term 2.
21:13:19.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:19.674225 6141 leader_election.cc:304] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: d65fad9630b3464aa5b089e635ea9fcb, dc7d413f28fc40929b53b58b87dc209e; no voters: dc9b40a769d748be83c1ae82b0e2a748
21:13:19.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:19.676596 6319 raft_consensus.cc:2804] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [term 2 FOLLOWER]: Leader election won for term 2
21:13:19.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:19.683290 6319 raft_consensus.cc:697] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [term 2 LEADER]: Becoming Leader. State: Replica: d65fad9630b3464aa5b089e635ea9fcb, State: Running, Role: LEADER
21:13:19.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:19.685379 6319 consensus_queue.cc:237] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc7d413f28fc40929b53b58b87dc209e" member_type: VOTER last_known_addr { host: "127.5.163.190" port: 35663 } } peers { permanent_uuid: "dc9b40a769d748be83c1ae82b0e2a748" member_type: VOTER last_known_addr { host: "127.5.163.189" port: 41991 } } peers { permanent_uuid: "d65fad9630b3464aa5b089e635ea9fcb" member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } }
21:13:19.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:19.707106 6324 sys_catalog.cc:455] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [sys.catalog]: SysCatalogTable state changed. Reason: New leader d65fad9630b3464aa5b089e635ea9fcb. Latest consensus state: current_term: 2 leader_uuid: "d65fad9630b3464aa5b089e635ea9fcb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc7d413f28fc40929b53b58b87dc209e" member_type: VOTER last_known_addr { host: "127.5.163.190" port: 35663 } } peers { permanent_uuid: "dc9b40a769d748be83c1ae82b0e2a748" member_type: VOTER last_known_addr { host: "127.5.163.189" port: 41991 } } peers { permanent_uuid: "d65fad9630b3464aa5b089e635ea9fcb" member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } } }
21:13:19.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:19.708118 6324 sys_catalog.cc:458] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [sys.catalog]: This master's current role is: LEADER
21:13:19.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:19.714473 6326 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
21:13:19.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:19.731953 6326 catalog_manager.cc:1494] Initializing Kudu cluster ID...
21:13:19.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:19.741030 6326 catalog_manager.cc:1269] Loaded cluster ID: 598f0d2c2b304c9784e7cb98a66bf96e
21:13:19.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:19.742518 6326 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
21:13:19.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:19.747200 6326 catalog_manager.cc:1514] Loading token signing keys...
21:13:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:19.759243 6326 catalog_manager.cc:6038] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb: Loaded TSK: 0
21:13:19.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:19.763123 6326 catalog_manager.cc:1524] Initializing in-progress tserver states...
21:13:20.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:20.108579 6142 consensus_peers.cc:597] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb -> Peer dc9b40a769d748be83c1ae82b0e2a748 (127.5.163.189:41991): Couldn't send request to peer dc9b40a769d748be83c1ae82b0e2a748. Status: Network error: Client connection negotiation failed: client connection to 127.5.163.189:41991: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:13:20.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:20.156771 6259 raft_consensus.cc:1275] T 00000000000000000000000000000000 P dc7d413f28fc40929b53b58b87dc209e [term 2 FOLLOWER]: Refusing update from remote peer d65fad9630b3464aa5b089e635ea9fcb: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
21:13:20.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:20.159487 6324 consensus_queue.cc:1048] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc7d413f28fc40929b53b58b87dc209e" member_type: VOTER last_known_addr { host: "127.5.163.190" port: 35663 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
21:13:20.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:20.212174 6319 sys_catalog.cc:455] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "d65fad9630b3464aa5b089e635ea9fcb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc7d413f28fc40929b53b58b87dc209e" member_type: VOTER last_known_addr { host: "127.5.163.190" port: 35663 } } peers { permanent_uuid: "dc9b40a769d748be83c1ae82b0e2a748" member_type: VOTER last_known_addr { host: "127.5.163.189" port: 41991 } } peers { permanent_uuid: "d65fad9630b3464aa5b089e635ea9fcb" member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } } }
21:13:20.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:20.212872 6319 sys_catalog.cc:458] T 00000000000000000000000000000000 P d65fad9630b3464aa5b089e635ea9fcb [sys.catalog]: This master's current role is: LEADER
21:13:20.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:20.220036 6333 sys_catalog.cc:455] T 00000000000000000000000000000000 P dc7d413f28fc40929b53b58b87dc209e [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "d65fad9630b3464aa5b089e635ea9fcb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc7d413f28fc40929b53b58b87dc209e" member_type: VOTER last_known_addr { host: "127.5.163.190" port: 35663 } } peers { permanent_uuid: "dc9b40a769d748be83c1ae82b0e2a748" member_type: VOTER last_known_addr { host: "127.5.163.189" port: 41991 } } peers { permanent_uuid: "d65fad9630b3464aa5b089e635ea9fcb" member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } } }
21:13:20.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:20.220741 6333 sys_catalog.cc:458] T 00000000000000000000000000000000 P dc7d413f28fc40929b53b58b87dc209e [sys.catalog]: This master's current role is: FOLLOWER
21:13:20.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:20.222743 6329 sys_catalog.cc:455] T 00000000000000000000000000000000 P dc7d413f28fc40929b53b58b87dc209e [sys.catalog]: SysCatalogTable state changed. Reason: New leader d65fad9630b3464aa5b089e635ea9fcb. Latest consensus state: current_term: 2 leader_uuid: "d65fad9630b3464aa5b089e635ea9fcb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc7d413f28fc40929b53b58b87dc209e" member_type: VOTER last_known_addr { host: "127.5.163.190" port: 35663 } } peers { permanent_uuid: "dc9b40a769d748be83c1ae82b0e2a748" member_type: VOTER last_known_addr { host: "127.5.163.189" port: 41991 } } peers { permanent_uuid: "d65fad9630b3464aa5b089e635ea9fcb" member_type: VOTER last_known_addr { host: "127.5.163.188" port: 32827 } } }
21:13:20.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:20.223562 6329 sys_catalog.cc:458] T 00000000000000000000000000000000 P dc7d413f28fc40929b53b58b87dc209e [sys.catalog]: This master's current role is: FOLLOWER
21:13:20.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:20.531994 6308 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 6286
21:13:20.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:19.146728 6309 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:13:20.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:20.753443 6286 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.610s user 0.574s sys 0.989s
21:13:20.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:20.755368 6286 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.612s user 0.574s sys 0.990s
21:13:20.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:20.756551 6312 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:13:20.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:20.758523 6311 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1614 milliseconds
21:13:20.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:20.758608 6286 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:13:20.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:20.760325 6286 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:13:20.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:20.763383 6286 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:13:20.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:20.764811 6286 hybrid_clock.cc:648] HybridClock initialized: now 1765487600764721 us; error 80 us; skew 500 ppm
21:13:20.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:20.766000 6286 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:13:20.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 11 21:13:20 dist-test-slave-2lgj krb5kdc[5786](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765487600, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.163.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:13:20.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:20.773197 6286 init.cc:377] Logged in from keytab as kudu/127.5.163.189@KRBTEST.COM (short username kudu)
21:13:20.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:20.778270 6286 webserver.cc:492] Webserver started at http://127.5.163.189:34781/ using document root <none> and password file <none>
21:13:20.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:20.779075 6286 fs_manager.cc:362] Metadata directory not provided
21:13:20.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:20.779271 6286 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:13:20.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:20.787117 6286 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.002s sys 0.002s
21:13:20.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:20.791528 6342 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:13:20.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:20.792586 6286 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
21:13:20.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:20.792896 6286 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-1/data,/tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-1/wal
21:13:20.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dc9b40a769d748be83c1ae82b0e2a748"
21:13:20.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-11 21:13:08 on dist-test-slave-2lgj"
21:13:20.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:20.794771 6286 fs_report.cc:389] FS layout report
21:13:20.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:13:20.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-1/wal
21:13:20.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-1/wal
21:13:20.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-1/data/data
21:13:20.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:13:20.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:13:20.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:13:20.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:13:20.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:13:20.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:13:20.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:13:20.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:13:20.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:13:20.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:13:20.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:13:20.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:20.847883 6286 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:13:20.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:20.853101 6286 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:13:20.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:20.853535 6286 kserver.cc:163] Server-wide thread pool size limit: 3276
21:13:20.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:20.933681 6286 rpc_server.cc:307] RPC server started. Bound to: 127.5.163.189:41991
21:13:20.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:20.933759 6394 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.5.163.189:41991 every 8 connection(s)
21:13:20.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:20.936899 6286 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/master-1/data/info.pb
21:13:20.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:20.943627 5774 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu as pid 6286
21:13:20.944 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:13:20.945 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.5.163.188:32827(127.5.163.188:32827)] connecting to peer
21:13:20.947 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.5.163.190:35663(127.5.163.190:35663)] connecting to peer
21:13:20.948 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.5.163.189:41991(127.5.163.189:41991)] connecting to peer
21:13:20.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:20.948812 6395 sys_catalog.cc:263] Verifying existing consensus state
21:13:20.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:20.956173 6395 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dc9b40a769d748be83c1ae82b0e2a748: Bootstrap starting.
21:13:20.964 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.5.163.190:35663(127.5.163.190:35663)] Successfully connected to peer
21:13:20.966 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.5.163.188:32827(127.5.163.188:32827)] Successfully connected to peer
21:13:20.970 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.5.163.189:41991(127.5.163.189:41991)] Successfully connected to peer
21:13:20.972 [DEBUG - kudu-nio-0] (SecurityContext.java:222) Refreshing Kerberos credentials...
21:13:20.972 [DEBUG - kudu-nio-0] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskpJdSDR/test-tmp/mini-kudu-cluster4098104208463714387/krb5kdc/krb5cc
21:13:20.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 11 21:13:20 dist-test-slave-2lgj krb5kdc[5786](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765487591, etypes {rep=17 tkt=17 ses=17}, test-user@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:13:20.981 [DEBUG - kudu-nio-0] (SecurityUtil.java:119) Logged in as subject: test-user@KRBTEST.COM
21:13:20.981 [ERROR - kudu-nio-0] (SecurityContext.java:246) Attempted to refresh Kerberos credentials from ticket cache but found that the new Kerberos principal test-user@KRBTEST.COM did not match the original principal test-admin@KRBTEST.COM. Ignoring.
21:13:20.987 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.5.163.188
21:13:20.993 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.5.163.190
21:13:21.010 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.163.188:32827(127.5.163.188:32827)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
21:13:21.017 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.163.188:32827: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
21:13:21.019 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.163.190:35663(127.5.163.190:35663)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
21:13:21.019 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.163.190:35663: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
21:13:21.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:21.020172 6401 negotiation.cc:337] Failed RPC negotiation. Trace:
21:13:21.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1211 21:13:20.952294 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:50762 (local address 127.5.163.188:32827)
21:13:21.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1211 21:13:20.968147 (+ 15853us) server_negotiation.cc:207] Beginning negotiation
21:13:21.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1211 21:13:20.968177 (+ 30us) server_negotiation.cc:400] Waiting for connection header
21:13:21.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1211 21:13:20.968236 (+ 59us) server_negotiation.cc:408] Connection header received
21:13:21.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1211 21:13:20.969996 (+ 1760us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
21:13:21.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1211 21:13:20.970028 (+ 32us) server_negotiation.cc:462] Received NEGOTIATE request from client
21:13:21.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1211 21:13:20.970377 (+ 349us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
21:13:21.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1211 21:13:20.970763 (+ 386us) server_negotiation.cc:227] Negotiated authn=SASL
21:13:21.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1211 21:13:20.994245 (+ 23482us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
21:13:21.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1211 21:13:20.996087 (+ 1842us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
21:13:21.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1211 21:13:21.006881 (+ 10794us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
21:13:21.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1211 21:13:21.007543 (+ 662us) server_negotiation.cc:663] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
21:13:21.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1211 21:13:21.018602 (+ 11059us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:50762: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:50762): Cannot send after transport endpoint shutdown (error 108)
21:13:21.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":15032,"thread_start_us":15028,"threads_started":1}
21:13:21.021 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.163.189:41991(127.5.163.189:41991)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
21:13:21.021 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.163.189:41991: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
21:13:21.022 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1523) Notify RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=77), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.163.188:32827, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.163.190:35663, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.163.189:41991, [73ms] Sub RPC ConnectToMaster: received response from server master-127.5.163.188:32827: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [75ms] Sub RPC ConnectToMaster: received response from server master-127.5.163.190:35663: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [77ms] Sub RPC ConnectToMaster: received response from server master-127.5.163.189:41991: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, deferred=null) after lookup exception
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.5.163.188:32827,127.5.163.190:35663,127.5.163.189:41991). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) [main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) [main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) [main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) [main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) [async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) [main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) [main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) [main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) [main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) [main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) [main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) [main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) [main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) [netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.5.163.188:32827,127.5.163.190:35663,127.5.163.189:41991). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
21:13:21.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:21.024399 6400 negotiation.cc:337] Failed RPC negotiation. Trace:
21:13:21.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1211 21:13:20.951743 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:41198 (local address 127.5.163.190:35663)
21:13:21.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1211 21:13:20.982388 (+ 30645us) server_negotiation.cc:207] Beginning negotiation
21:13:21.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1211 21:13:20.982416 (+ 28us) server_negotiation.cc:400] Waiting for connection header
21:13:21.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1211 21:13:20.982467 (+ 51us) server_negotiation.cc:408] Connection header received
21:13:21.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1211 21:13:20.982609 (+ 142us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
21:13:21.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1211 21:13:20.982636 (+ 27us) server_negotiation.cc:462] Received NEGOTIATE request from client
21:13:21.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1211 21:13:20.982855 (+ 219us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
21:13:21.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1211 21:13:20.983067 (+ 212us) server_negotiation.cc:227] Negotiated authn=SASL
21:13:21.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1211 21:13:20.997286 (+ 14219us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
21:13:21.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1211 21:13:20.999165 (+ 1879us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
21:13:21.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1211 21:13:21.013264 (+ 14099us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
21:13:21.025 [DEBUG - kudu-nio-0] (KuduRpc.java:260) Handling a callback on RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=80), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.163.188:32827, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.163.190:35663, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.163.189:41991, [73ms] Sub RPC ConnectToMaster: received response from server master-127.5.163.188:32827: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [75ms] Sub RPC ConnectToMaster: received response from server master-127.5.163.190:35663: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [77ms] Sub RPC ConnectToMaster: received response from server master-127.5.163.189:41991: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, deferred=null) with no deferred attached!
21:13:21.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1211 21:13:21.013890 (+ 626us) server_negotiation.cc:663] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
21:13:21.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1211 21:13:21.022808 (+ 8918us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:41198: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:41198): Cannot send after transport endpoint shutdown (error 108)
21:13:21.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":29824,"thread_start_us":29984,"threads_started":1}
21:13:21.025 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.5.163.188:32827(127.5.163.188:32827)] handling channelInactive
21:13:21.026 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.5.163.189:41991(127.5.163.189:41991)] handling channelInactive
21:13:21.031 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.5.163.190:35663(127.5.163.190:35663)] handling channelInactive
21:13:21.031 [DEBUG - main] (Connection.java:575) [peer master-127.5.163.188:32827(127.5.163.188:32827)] disconnecting while in state TERMINATED
21:13:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:21.027803 6402 negotiation.cc:337] Failed RPC negotiation. Trace:
21:13:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1211 21:13:20.960691 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:40056 (local address 127.5.163.189:41991)
21:13:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1211 21:13:20.983207 (+ 22516us) server_negotiation.cc:207] Beginning negotiation
21:13:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1211 21:13:20.983247 (+ 40us) server_negotiation.cc:400] Waiting for connection header
21:13:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1211 21:13:20.983316 (+ 69us) server_negotiation.cc:408] Connection header received
21:13:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1211 21:13:20.991158 (+ 7842us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
21:13:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1211 21:13:20.991214 (+ 56us) server_negotiation.cc:462] Received NEGOTIATE request from client
21:13:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1211 21:13:20.991691 (+ 477us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
21:13:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1211 21:13:20.992119 (+ 428us) server_negotiation.cc:227] Negotiated authn=SASL
21:13:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1211 21:13:21.026121 (+ 34002us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:40056: BlockingRecv error: recv got EOF from 127.0.0.1:40056 (error 108)
21:13:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":21817,"thread_start_us":22468,"threads_started":1}
21:13:21.033 [DEBUG - main] (Connection.java:575) [peer master-127.5.163.189:41991(127.5.163.189:41991)] disconnecting while in state TERMINATED
21:13:21.033 [DEBUG - main] (Connection.java:575) [peer master-127.5.163.190:35663(127.5.163.190:35663)] disconnecting while in state TERMINATED
21:13:21.034 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
21:13:21.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:21.035522 5774 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu with pid 6194
21:13:21.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:21.036661 6395 log.cc:826] T 00000000000000000000000000000000 P dc9b40a769d748be83c1ae82b0e2a748: Log is configured to *not* fsync() on all Append() calls
21:13:21.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:21.056159 6395 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dc9b40a769d748be83c1ae82b0e2a748: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:13:21.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:21.057147 6395 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dc9b40a769d748be83c1ae82b0e2a748: Bootstrap complete.
21:13:21.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:21.058358 5774 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu with pid 6286
21:13:21.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:21.073408 5774 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu with pid 6120
21:13:22.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-11T21:13:22Z chronyd exiting
21:13:23.223 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
21:13:23.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:23.225327 5048 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu with pid 5341
21:13:23.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251211 21:13:23.245200 5233 connection.cc:537] server connection from 127.4.238.1:49005 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
21:13:23.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:23.245641 5048 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu with pid 5491
21:13:23.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:23.265311 5048 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu with pid 5633
21:13:23.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:23.286274 5048 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu with pid 5060
21:13:23.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:23.311534 5048 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu with pid 5130
21:13:23.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251211 21:13:23.333662 5048 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskpJdSDR/build/tsan/bin/kudu with pid 5205
21:13:24.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-11T21:13:24Z chronyd exiting
21:13:25.410 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testDoNotSwitchPrincipalsInExistingClient: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.5.163.188:32827,127.5.163.190:35663,127.5.163.189:41991). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386) ~[test/:?]
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40) ~[kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testDoNotSwitchPrincipalsInExistingClient(TestSecurity.java:366) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.5.163.188:32827,127.5.163.190:35663,127.5.163.189:41991). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386)
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40)
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378)
at org.apache.kudu.client.TestSecurity.testDoNotSwitchPrincipalsInExistingClient(TestSecurity.java:366)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more