09:13:40.536 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
09:13:40.536 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskWfs5gY/test-tmp
09:13:40.536 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
09:13:40.536 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskWfs5gY/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
09:13:40.543 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
09:13:40.837 [DEBUG - main] (MiniKuduCluster.java:195) Response:
09:13:40.837 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
09:13:40.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:40.840078 23805 mini_oidc.cc:150] Starting JWKS server
09:13:40.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:40.843119 23805 webserver.cc:492] Webserver started at http://127.0.0.1:35663/ using document root <none> and password file <none>
09:13:40.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:40.843492 23805 mini_oidc.cc:186] Starting OIDC Discovery server
09:13:40.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:40.845049 23805 webserver.cc:492] Webserver started at http://127.0.0.1:45149/ using document root <none> and password file <none>
09:13:40.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-27T09:13:40Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
09:13:40.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-27T09:13:40Z Disabled control of system clock
09:13:40.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:40.898896 23805 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu
09:13:40.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu
09:13:40.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-0/wal
09:13:40.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-0/data
09:13:40.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:13:40.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:13:40.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:13:40.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:13:40.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:13:40.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:13:40.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-0/logs
09:13:40.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-0/data/info.pb
09:13:40.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:13:40.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:13:40.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:13:40.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:13:40.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:13:40.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:13:40.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:13:40.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:13:40.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:13:40.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:13:40.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:13:40.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:13:40.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.63.126:45639
09:13:40.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.63.126
09:13:40.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:13:40.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.63.84:45099
09:13:40.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:13:40.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:13:40.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:13:40.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:13:40.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.63.126:45639,127.23.63.125:34901,127.23.63.124:43733
09:13:40.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:13:40.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35663/jwks.json with env {}
09:13:41.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:41.164336 23817 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:13:41.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:41.164804 23817 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:13:41.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:41.165251 23817 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:13:41.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:41.195106 23817 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:13:41.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:41.195408 23817 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:13:41.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:41.195622 23817 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:13:41.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:41.195842 23817 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:13:41.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:41.196046 23817 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:13:41.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:41.196187 23817 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35663/jwks.json
09:13:41.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:41.226425 23817 master_runner.cc:386] Master server non-default flags:
09:13:41.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:13:41.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.63.84:45099
09:13:41.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:13:41.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:13:41.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-0/data
09:13:41.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-0/wal
09:13:41.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:13:41.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.63.126:45639,127.23.63.125:34901,127.23.63.124:43733
09:13:41.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:13:41.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:13:41.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:13:41.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.63.126:45639
09:13:41.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:13:41.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:13:41.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:13:41.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35663/jwks.json
09:13:41.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:13:41.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:13:41.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-0/data/info.pb
09:13:41.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.63.126
09:13:41.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:13:41.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:13:41.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:13:41.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:13:41.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:13:41.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:13:41.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-0/logs
09:13:41.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:13:41.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:13:41.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:13:41.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:13:41.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:13:41.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d0cb013a5aad754f2ed06c25f5ec3d8a1873622
09:13:41.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:13:41.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Oct 2025 08:43:21 UTC on 5fd53c4cbb9d
09:13:41.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8741
09:13:41.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:13:41.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:41.227499 23817 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:13:41.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:41.228947 23817 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:13:41.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:41.237915 23823 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:42.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:41.238976 23824 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:42.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:41.242343 23826 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:42.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:42.328987 23825 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1086 milliseconds
09:13:42.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:42.329107 23817 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:13:42.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:42.330246 23817 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:13:42.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:42.332517 23817 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:13:42.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:42.333827 23817 hybrid_clock.cc:648] HybridClock initialized: now 1761556422333779 us; error 53 us; skew 500 ppm
09:13:42.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:42.334574 23817 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:13:42.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:42.340337 23817 webserver.cc:492] Webserver started at http://127.23.63.126:39623/ using document root <none> and password file <none>
09:13:42.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:42.341264 23817 fs_manager.cc:362] Metadata directory not provided
09:13:42.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:42.341470 23817 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:13:42.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:42.341954 23817 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
09:13:42.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:42.345760 23817 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-0/data/instance:
09:13:42.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "771c7c9f49674c838e35b35ae30ef674"
09:13:42.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-27 09:13:42 on dist-test-slave-n41w"
09:13:42.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:42.346699 23817 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-0/wal/instance:
09:13:42.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "771c7c9f49674c838e35b35ae30ef674"
09:13:42.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-27 09:13:42 on dist-test-slave-n41w"
09:13:42.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:42.353262 23817 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.004s sys 0.004s
09:13:42.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:42.358281 23833 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:13:42.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:42.359207 23817 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
09:13:42.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:42.359475 23817 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-0/data,/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-0/wal
09:13:42.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "771c7c9f49674c838e35b35ae30ef674"
09:13:42.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-27 09:13:42 on dist-test-slave-n41w"
09:13:42.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:42.359826 23817 fs_report.cc:389] FS layout report
09:13:42.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:13:42.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-0/wal
09:13:42.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-0/wal
09:13:42.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-0/data/data
09:13:42.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:13:42.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:13:42.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:13:42.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:13:42.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:13:42.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:13:42.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:13:42.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:13:42.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:13:42.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:13:42.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:13:42.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:42.433508 23817 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:13:42.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:42.434691 23817 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:13:42.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:42.435086 23817 kserver.cc:163] Server-wide thread pool size limit: 3276
09:13:42.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:42.498908 23817 rpc_server.cc:307] RPC server started. Bound to: 127.23.63.126:45639
09:13:42.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:42.498967 23884 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.63.126:45639 every 8 connection(s)
09:13:42.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:42.501340 23817 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-0/data/info.pb
09:13:42.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:42.506659 23885 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
09:13:42.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:42.510797 23805 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu as pid 23817
09:13:42.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:42.511159 23805 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-0/wal/instance
09:13:42.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:42.513989 23805 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu
09:13:42.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu
09:13:42.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-1/wal
09:13:42.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-1/data
09:13:42.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:13:42.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:13:42.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:13:42.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:13:42.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:13:42.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:13:42.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-1/logs
09:13:42.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-1/data/info.pb
09:13:42.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:13:42.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:13:42.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:13:42.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:13:42.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:13:42.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:13:42.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:13:42.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:13:42.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:13:42.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:13:42.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:13:42.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:13:42.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.63.125:34901
09:13:42.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.63.125
09:13:42.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:13:42.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.63.84:45099
09:13:42.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:13:42.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:13:42.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:13:42.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:13:42.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.63.126:45639,127.23.63.125:34901,127.23.63.124:43733
09:13:42.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:13:42.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35663/jwks.json with env {}
09:13:42.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:42.516677 23885 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.63.126" port: 45639 } has no permanent_uuid. Determining permanent_uuid...
09:13:42.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:42.543555 23885 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.63.125" port: 34901 } has no permanent_uuid. Determining permanent_uuid...
09:13:42.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:42.545682 23834 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.63.125:34901: connect: Connection refused (error 111)
09:13:42.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:42.549237 23885 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.63.125:34901: Network error: Client connection negotiation failed: client connection to 127.23.63.125:34901: connect: Connection refused (error 111)
09:13:42.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:42.598624 23885 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.63.125" port: 34901 } attempt: 1
09:13:42.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:42.602659 23885 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.63.125:34901: Network error: Client connection negotiation failed: client connection to 127.23.63.125:34901: connect: Connection refused (error 111)
09:13:42.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:42.671066 23885 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.63.125" port: 34901 } attempt: 2
09:13:42.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:42.674691 23885 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.63.125:34901: Network error: Client connection negotiation failed: client connection to 127.23.63.125:34901: connect: Connection refused (error 111)
09:13:42.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:42.766059 23885 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.63.125" port: 34901 } attempt: 3
09:13:42.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:42.769941 23885 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.63.125:34901: Network error: Client connection negotiation failed: client connection to 127.23.63.125:34901: connect: Connection refused (error 111)
09:13:42.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:42.788668 23887 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:13:42.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:42.789252 23887 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:13:42.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:42.789907 23887 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:13:42.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:42.820343 23887 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:13:42.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:42.820686 23887 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:13:42.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:42.820982 23887 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:13:42.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:42.821251 23887 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:13:42.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:42.821520 23887 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:13:42.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:42.821729 23887 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35663/jwks.json
09:13:42.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:42.854454 23887 master_runner.cc:386] Master server non-default flags:
09:13:42.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:13:42.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.63.84:45099
09:13:42.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:13:42.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:13:42.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-1/data
09:13:42.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-1/wal
09:13:42.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:13:42.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.63.126:45639,127.23.63.125:34901,127.23.63.124:43733
09:13:42.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:13:42.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:13:42.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:13:42.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.63.125:34901
09:13:42.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:13:42.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:13:42.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:13:42.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35663/jwks.json
09:13:42.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:13:42.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:13:42.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-1/data/info.pb
09:13:42.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.63.125
09:13:42.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:13:42.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:13:42.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:13:42.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:13:42.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:13:42.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:13:42.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-1/logs
09:13:42.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:13:42.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:13:42.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:13:42.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:13:42.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:13:42.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d0cb013a5aad754f2ed06c25f5ec3d8a1873622
09:13:42.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:13:42.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Oct 2025 08:43:21 UTC on 5fd53c4cbb9d
09:13:42.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8741
09:13:42.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:13:42.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:42.856000 23887 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:13:42.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:42.857700 23887 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:13:42.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:42.869235 23896 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:42.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:42.913298 23885 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.63.125" port: 34901 } attempt: 4
09:13:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:42.924202 23885 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.63.125:34901: Network error: Client connection negotiation failed: client connection to 127.23.63.125:34901: connect: Connection refused (error 111)
09:13:43.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:43.223621 23885 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.63.125" port: 34901 } attempt: 5
09:13:43.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:43.230007 23885 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.63.125:34901: Network error: Client connection negotiation failed: client connection to 127.23.63.125:34901: connect: Connection refused (error 111)
09:13:43.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:43.777457 23885 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.63.125" port: 34901 } attempt: 6
09:13:43.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:43.806674 23885 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.63.125:34901: Network error: Client connection negotiation failed: client connection to 127.23.63.125:34901: connect: Connection refused (error 111)
09:13:44.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:42.869531 23897 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:44.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:44.236748 23899 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:44.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:44.239337 23898 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1366 milliseconds
09:13:44.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:44.239840 23887 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.371s user 0.434s sys 0.936s
09:13:44.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:44.240082 23887 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.371s user 0.434s sys 0.936s
09:13:44.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:44.240307 23887 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:13:44.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:44.241281 23887 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:13:44.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:44.246393 23887 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:13:44.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:44.247706 23887 hybrid_clock.cc:648] HybridClock initialized: now 1761556424247655 us; error 52 us; skew 500 ppm
09:13:44.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:44.248445 23887 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:13:44.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:44.254864 23887 webserver.cc:492] Webserver started at http://127.23.63.125:42915/ using document root <none> and password file <none>
09:13:44.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:44.255686 23887 fs_manager.cc:362] Metadata directory not provided
09:13:44.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:44.255880 23887 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:13:44.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:44.256268 23887 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
09:13:44.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:44.260068 23887 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-1/data/instance:
09:13:44.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ed9a912ea092472eae7d7af380068f95"
09:13:44.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-27 09:13:44 on dist-test-slave-n41w"
09:13:44.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:44.261035 23887 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-1/wal/instance:
09:13:44.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ed9a912ea092472eae7d7af380068f95"
09:13:44.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-27 09:13:44 on dist-test-slave-n41w"
09:13:44.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:44.267879 23887 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.005s sys 0.002s
09:13:44.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:44.273392 23908 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:13:44.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:44.274369 23887 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
09:13:44.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:44.274637 23887 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-1/data,/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-1/wal
09:13:44.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ed9a912ea092472eae7d7af380068f95"
09:13:44.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-27 09:13:44 on dist-test-slave-n41w"
09:13:44.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:44.274940 23887 fs_report.cc:389] FS layout report
09:13:44.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:13:44.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-1/wal
09:13:44.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-1/wal
09:13:44.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-1/data/data
09:13:44.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:13:44.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:13:44.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:13:44.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:13:44.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:13:44.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:13:44.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:13:44.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:13:44.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:13:44.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:13:44.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:13:44.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:44.347056 23887 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:13:44.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:44.348344 23887 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:13:44.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:44.348738 23887 kserver.cc:163] Server-wide thread pool size limit: 3276
09:13:44.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:44.411301 23887 rpc_server.cc:307] RPC server started. Bound to: 127.23.63.125:34901
09:13:44.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:44.411353 23959 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.63.125:34901 every 8 connection(s)
09:13:44.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:44.413755 23887 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-1/data/info.pb
09:13:44.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:44.419262 23960 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
09:13:44.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:44.422286 23805 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu as pid 23887
09:13:44.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:44.422638 23805 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-1/wal/instance
09:13:44.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:44.424372 23805 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu
09:13:44.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu
09:13:44.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-2/wal
09:13:44.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-2/data
09:13:44.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:13:44.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:13:44.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:13:44.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:13:44.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:13:44.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:13:44.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-2/logs
09:13:44.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-2/data/info.pb
09:13:44.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:13:44.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:13:44.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:13:44.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:13:44.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:13:44.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:13:44.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:13:44.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:13:44.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:13:44.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:13:44.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:13:44.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:13:44.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.63.124:43733
09:13:44.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.63.124
09:13:44.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:13:44.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.63.84:45099
09:13:44.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:13:44.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:13:44.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:13:44.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:13:44.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.63.126:45639,127.23.63.125:34901,127.23.63.124:43733
09:13:44.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:13:44.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35663/jwks.json with env {}
09:13:44.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:44.428195 23960 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.63.126" port: 45639 } has no permanent_uuid. Determining permanent_uuid...
09:13:44.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:44.446703 23960 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.63.125" port: 34901 } has no permanent_uuid. Determining permanent_uuid...
09:13:44.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:44.460747 23960 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.63.124" port: 43733 } has no permanent_uuid. Determining permanent_uuid...
09:13:44.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:44.462803 23911 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.63.124:43733: connect: Connection refused (error 111)
09:13:44.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:44.466677 23960 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.63.124:43733: Network error: Client connection negotiation failed: client connection to 127.23.63.124:43733: connect: Connection refused (error 111)
09:13:44.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:44.516093 23960 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.63.124" port: 43733 } attempt: 1
09:13:44.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:44.520056 23960 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.63.124:43733: Network error: Client connection negotiation failed: client connection to 127.23.63.124:43733: connect: Connection refused (error 111)
09:13:44.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:44.588367 23960 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.63.124" port: 43733 } attempt: 2
09:13:44.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:44.591749 23960 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.63.124:43733: Network error: Client connection negotiation failed: client connection to 127.23.63.124:43733: connect: Connection refused (error 111)
09:13:44.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:44.683176 23960 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.63.124" port: 43733 } attempt: 3
09:13:44.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:44.686465 23960 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.63.124:43733: Network error: Client connection negotiation failed: client connection to 127.23.63.124:43733: connect: Connection refused (error 111)
09:13:44.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:44.700830 23962 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:13:44.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:44.701236 23962 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:13:44.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:44.701622 23962 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:13:44.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:44.728250 23962 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:13:44.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:44.728464 23962 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:13:44.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:44.728643 23962 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:13:44.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:44.728830 23962 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:13:44.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:44.728989 23962 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:13:44.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:44.729110 23962 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35663/jwks.json
09:13:44.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:44.758360 23962 master_runner.cc:386] Master server non-default flags:
09:13:44.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:13:44.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.63.84:45099
09:13:44.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:13:44.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:13:44.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-2/data
09:13:44.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-2/wal
09:13:44.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:13:44.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.63.126:45639,127.23.63.125:34901,127.23.63.124:43733
09:13:44.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:13:44.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:13:44.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:13:44.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.63.124:43733
09:13:44.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:13:44.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:13:44.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:13:44.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35663/jwks.json
09:13:44.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:13:44.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:13:44.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-2/data/info.pb
09:13:44.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.63.124
09:13:44.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:13:44.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:13:44.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:13:44.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:13:44.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:13:44.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:13:44.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-2/logs
09:13:44.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:13:44.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:13:44.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:13:44.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:13:44.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:13:44.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d0cb013a5aad754f2ed06c25f5ec3d8a1873622
09:13:44.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:13:44.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Oct 2025 08:43:21 UTC on 5fd53c4cbb9d
09:13:44.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8741
09:13:44.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:13:44.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:44.759389 23962 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:13:44.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:44.760766 23962 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:13:44.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:44.769655 23972 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:44.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:44.829859 23960 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.63.124" port: 43733 } attempt: 4
09:13:44.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:44.836010 23960 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.63.124:43733: Network error: Client connection negotiation failed: client connection to 127.23.63.124:43733: connect: Connection refused (error 111)
09:13:44.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:44.867159 23885 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.63.125" port: 34901 } attempt: 7
09:13:44.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:44.894943 23885 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.63.124" port: 43733 } has no permanent_uuid. Determining permanent_uuid...
09:13:44.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:44.913977 23885 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.63.124:43733: Network error: Client connection negotiation failed: client connection to 127.23.63.124:43733: connect: Connection refused (error 111)
09:13:44.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:44.972414 23885 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.63.124" port: 43733 } attempt: 1
09:13:44.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:44.978295 23885 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.63.124:43733: Network error: Client connection negotiation failed: client connection to 127.23.63.124:43733: connect: Connection refused (error 111)
09:13:45.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:45.059759 23885 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.63.124" port: 43733 } attempt: 2
09:13:45.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:45.065666 23885 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.63.124:43733: Network error: Client connection negotiation failed: client connection to 127.23.63.124:43733: connect: Connection refused (error 111)
09:13:45.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:45.135461 23960 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.63.124" port: 43733 } attempt: 5
09:13:45.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:45.141628 23960 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.63.124:43733: Network error: Client connection negotiation failed: client connection to 127.23.63.124:43733: connect: Connection refused (error 111)
09:13:45.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:45.151116 23885 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.63.124" port: 43733 } attempt: 3
09:13:45.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:45.156863 23885 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.63.124:43733: Network error: Client connection negotiation failed: client connection to 127.23.63.124:43733: connect: Connection refused (error 111)
09:13:45.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:45.297343 23885 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.63.124" port: 43733 } attempt: 4
09:13:45.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:45.304520 23885 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.63.124:43733: Network error: Client connection negotiation failed: client connection to 127.23.63.124:43733: connect: Connection refused (error 111)
09:13:45.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:45.587961 23885 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.63.124" port: 43733 } attempt: 5
09:13:45.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:45.594678 23885 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.63.124:43733: Network error: Client connection negotiation failed: client connection to 127.23.63.124:43733: connect: Connection refused (error 111)
09:13:45.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:45.689100 23960 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.63.124" port: 43733 } attempt: 6
09:13:45.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:45.710467 23960 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.63.124:43733: Network error: Client connection negotiation failed: client connection to 127.23.63.124:43733: connect: Connection refused (error 111)
09:13:46.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:44.770052 23973 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:46.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:46.078785 23962 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.309s user 0.507s sys 0.794s
09:13:46.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:46.079174 23962 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.309s user 0.507s sys 0.794s
09:13:46.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:46.080099 23974 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1305 milliseconds
09:13:46.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:46.080197 23975 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:46.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.080206 23962 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:13:46.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.081766 23962 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:13:46.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.084681 23962 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:13:46.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.086144 23962 hybrid_clock.cc:648] HybridClock initialized: now 1761556426086026 us; error 104 us; skew 500 ppm
09:13:46.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.087239 23962 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:13:46.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.095366 23962 webserver.cc:492] Webserver started at http://127.23.63.124:42007/ using document root <none> and password file <none>
09:13:46.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.096678 23962 fs_manager.cc:362] Metadata directory not provided
09:13:46.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.096971 23962 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:13:46.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.097503 23962 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
09:13:46.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.103703 23962 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-2/data/instance:
09:13:46.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2ea1d9e208a44e998b21c0d679d4efc7"
09:13:46.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-27 09:13:46 on dist-test-slave-n41w"
09:13:46.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.105283 23962 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-2/wal/instance:
09:13:46.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2ea1d9e208a44e998b21c0d679d4efc7"
09:13:46.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-27 09:13:46 on dist-test-slave-n41w"
09:13:46.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.114321 23962 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.008s sys 0.000s
09:13:46.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.121559 23986 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:13:46.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.122493 23962 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
09:13:46.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.122738 23962 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-2/data,/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-2/wal
09:13:46.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2ea1d9e208a44e998b21c0d679d4efc7"
09:13:46.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-27 09:13:46 on dist-test-slave-n41w"
09:13:46.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.122994 23962 fs_report.cc:389] FS layout report
09:13:46.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:13:46.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-2/wal
09:13:46.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-2/wal
09:13:46.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-2/data/data
09:13:46.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:13:46.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:13:46.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:13:46.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:13:46.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:13:46.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:13:46.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:13:46.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:13:46.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:13:46.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:13:46.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:13:46.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.147104 23885 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.63.124" port: 43733 } attempt: 6
09:13:46.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:46.153877 23885 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.63.124:43733: Network error: Client connection negotiation failed: client connection to 127.23.63.124:43733: connect: Connection refused (error 111)
09:13:46.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.185806 23962 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:13:46.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.186980 23962 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:13:46.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.187351 23962 kserver.cc:163] Server-wide thread pool size limit: 3276
09:13:46.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.247449 23962 rpc_server.cc:307] RPC server started. Bound to: 127.23.63.124:43733
09:13:46.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.247525 24039 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.63.124:43733 every 8 connection(s)
09:13:46.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.249809 23962 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-2/data/info.pb
09:13:46.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.254609 24040 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
09:13:46.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.256265 23805 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu as pid 23962
09:13:46.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.256546 23805 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/master-2/wal/instance
09:13:46.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.268071 24040 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.63.126" port: 45639 } has no permanent_uuid. Determining permanent_uuid...
09:13:46.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.289485 24040 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.63.125" port: 34901 } has no permanent_uuid. Determining permanent_uuid...
09:13:46.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.297456 24040 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.63.124" port: 43733 } has no permanent_uuid. Determining permanent_uuid...
09:13:46.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.310287 24040 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2ea1d9e208a44e998b21c0d679d4efc7: Bootstrap starting.
09:13:46.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.315238 24040 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 2ea1d9e208a44e998b21c0d679d4efc7: Neither blocks nor log segments found. Creating new log.
09:13:46.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.316998 24040 log.cc:826] T 00000000000000000000000000000000 P 2ea1d9e208a44e998b21c0d679d4efc7: Log is configured to *not* fsync() on all Append() calls
09:13:46.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.320588 24040 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2ea1d9e208a44e998b21c0d679d4efc7: No bootstrap required, opened a new log
09:13:46.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.336144 24040 raft_consensus.cc:359] T 00000000000000000000000000000000 P 2ea1d9e208a44e998b21c0d679d4efc7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "771c7c9f49674c838e35b35ae30ef674" member_type: VOTER last_known_addr { host: "127.23.63.126" port: 45639 } } peers { permanent_uuid: "ed9a912ea092472eae7d7af380068f95" member_type: VOTER last_known_addr { host: "127.23.63.125" port: 34901 } } peers { permanent_uuid: "2ea1d9e208a44e998b21c0d679d4efc7" member_type: VOTER last_known_addr { host: "127.23.63.124" port: 43733 } }
09:13:46.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.336606 24040 raft_consensus.cc:385] T 00000000000000000000000000000000 P 2ea1d9e208a44e998b21c0d679d4efc7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:13:46.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.336768 24040 raft_consensus.cc:740] T 00000000000000000000000000000000 P 2ea1d9e208a44e998b21c0d679d4efc7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2ea1d9e208a44e998b21c0d679d4efc7, State: Initialized, Role: FOLLOWER
09:13:46.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.337255 24040 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2ea1d9e208a44e998b21c0d679d4efc7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "771c7c9f49674c838e35b35ae30ef674" member_type: VOTER last_known_addr { host: "127.23.63.126" port: 45639 } } peers { permanent_uuid: "ed9a912ea092472eae7d7af380068f95" member_type: VOTER last_known_addr { host: "127.23.63.125" port: 34901 } } peers { permanent_uuid: "2ea1d9e208a44e998b21c0d679d4efc7" member_type: VOTER last_known_addr { host: "127.23.63.124" port: 43733 } }
09:13:46.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.339180 24049 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2ea1d9e208a44e998b21c0d679d4efc7 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "771c7c9f49674c838e35b35ae30ef674" member_type: VOTER last_known_addr { host: "127.23.63.126" port: 45639 } } peers { permanent_uuid: "ed9a912ea092472eae7d7af380068f95" member_type: VOTER last_known_addr { host: "127.23.63.125" port: 34901 } } peers { permanent_uuid: "2ea1d9e208a44e998b21c0d679d4efc7" member_type: VOTER last_known_addr { host: "127.23.63.124" port: 43733 } } }
09:13:46.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.339706 24049 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2ea1d9e208a44e998b21c0d679d4efc7 [sys.catalog]: This master's current role is: FOLLOWER
09:13:46.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.340394 24040 sys_catalog.cc:565] T 00000000000000000000000000000000 P 2ea1d9e208a44e998b21c0d679d4efc7 [sys.catalog]: configured and running, proceeding with master startup.
09:13:46.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:46.361200 24060 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 2ea1d9e208a44e998b21c0d679d4efc7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
09:13:46.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:46.361482 24060 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
09:13:46.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.357635 23962 master_runner.cc:186] Error getting master registration for 127.23.63.126:45639: OK, instance_id { permanent_uuid: "771c7c9f49674c838e35b35ae30ef674" instance_seqno: 1761556422481599 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
09:13:46.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.376582 23962 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
09:13:46.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:46.441805 23994 tablet.cc:2378] T 00000000000000000000000000000000 P 2ea1d9e208a44e998b21c0d679d4efc7: Can't schedule compaction. Clean time has not been advanced past its initial value.
09:13:46.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.506884 24049 raft_consensus.cc:493] T 00000000000000000000000000000000 P 2ea1d9e208a44e998b21c0d679d4efc7 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:13:46.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.507382 24049 raft_consensus.cc:515] T 00000000000000000000000000000000 P 2ea1d9e208a44e998b21c0d679d4efc7 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "771c7c9f49674c838e35b35ae30ef674" member_type: VOTER last_known_addr { host: "127.23.63.126" port: 45639 } } peers { permanent_uuid: "ed9a912ea092472eae7d7af380068f95" member_type: VOTER last_known_addr { host: "127.23.63.125" port: 34901 } } peers { permanent_uuid: "2ea1d9e208a44e998b21c0d679d4efc7" member_type: VOTER last_known_addr { host: "127.23.63.124" port: 43733 } }
09:13:46.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.509757 24049 leader_election.cc:290] T 00000000000000000000000000000000 P 2ea1d9e208a44e998b21c0d679d4efc7 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 771c7c9f49674c838e35b35ae30ef674 (127.23.63.126:45639), ed9a912ea092472eae7d7af380068f95 (127.23.63.125:34901)
09:13:46.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.770925 23960 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.63.124" port: 43733 } attempt: 7
09:13:46.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.800661 23960 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ed9a912ea092472eae7d7af380068f95: Bootstrap starting.
09:13:46.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.807408 23960 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ed9a912ea092472eae7d7af380068f95: Neither blocks nor log segments found. Creating new log.
09:13:46.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.809512 23960 log.cc:826] T 00000000000000000000000000000000 P ed9a912ea092472eae7d7af380068f95: Log is configured to *not* fsync() on all Append() calls
09:13:46.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.813526 23960 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ed9a912ea092472eae7d7af380068f95: No bootstrap required, opened a new log
09:13:46.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.831022 23960 raft_consensus.cc:359] T 00000000000000000000000000000000 P ed9a912ea092472eae7d7af380068f95 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "771c7c9f49674c838e35b35ae30ef674" member_type: VOTER last_known_addr { host: "127.23.63.126" port: 45639 } } peers { permanent_uuid: "ed9a912ea092472eae7d7af380068f95" member_type: VOTER last_known_addr { host: "127.23.63.125" port: 34901 } } peers { permanent_uuid: "2ea1d9e208a44e998b21c0d679d4efc7" member_type: VOTER last_known_addr { host: "127.23.63.124" port: 43733 } }
09:13:46.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.831482 23960 raft_consensus.cc:385] T 00000000000000000000000000000000 P ed9a912ea092472eae7d7af380068f95 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:13:46.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.831722 23960 raft_consensus.cc:740] T 00000000000000000000000000000000 P ed9a912ea092472eae7d7af380068f95 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ed9a912ea092472eae7d7af380068f95, State: Initialized, Role: FOLLOWER
09:13:46.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.832311 23960 consensus_queue.cc:260] T 00000000000000000000000000000000 P ed9a912ea092472eae7d7af380068f95 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "771c7c9f49674c838e35b35ae30ef674" member_type: VOTER last_known_addr { host: "127.23.63.126" port: 45639 } } peers { permanent_uuid: "ed9a912ea092472eae7d7af380068f95" member_type: VOTER last_known_addr { host: "127.23.63.125" port: 34901 } } peers { permanent_uuid: "2ea1d9e208a44e998b21c0d679d4efc7" member_type: VOTER last_known_addr { host: "127.23.63.124" port: 43733 } }
09:13:46.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.834811 24064 sys_catalog.cc:455] T 00000000000000000000000000000000 P ed9a912ea092472eae7d7af380068f95 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "771c7c9f49674c838e35b35ae30ef674" member_type: VOTER last_known_addr { host: "127.23.63.126" port: 45639 } } peers { permanent_uuid: "ed9a912ea092472eae7d7af380068f95" member_type: VOTER last_known_addr { host: "127.23.63.125" port: 34901 } } peers { permanent_uuid: "2ea1d9e208a44e998b21c0d679d4efc7" member_type: VOTER last_known_addr { host: "127.23.63.124" port: 43733 } } }
09:13:46.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.835501 24064 sys_catalog.cc:458] T 00000000000000000000000000000000 P ed9a912ea092472eae7d7af380068f95 [sys.catalog]: This master's current role is: FOLLOWER
09:13:46.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.836390 23960 sys_catalog.cc:565] T 00000000000000000000000000000000 P ed9a912ea092472eae7d7af380068f95 [sys.catalog]: configured and running, proceeding with master startup.
09:13:46.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.836597 23935 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2ea1d9e208a44e998b21c0d679d4efc7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ed9a912ea092472eae7d7af380068f95" is_pre_election: true
09:13:46.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.837334 23935 raft_consensus.cc:2468] T 00000000000000000000000000000000 P ed9a912ea092472eae7d7af380068f95 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2ea1d9e208a44e998b21c0d679d4efc7 in term 0.
09:13:46.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.838649 23989 leader_election.cc:304] T 00000000000000000000000000000000 P 2ea1d9e208a44e998b21c0d679d4efc7 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2ea1d9e208a44e998b21c0d679d4efc7, ed9a912ea092472eae7d7af380068f95; no voters:
09:13:46.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.839489 24049 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 2ea1d9e208a44e998b21c0d679d4efc7 [term 0 FOLLOWER]: Leader pre-election won for term 1
09:13:46.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.839854 24049 raft_consensus.cc:493] T 00000000000000000000000000000000 P 2ea1d9e208a44e998b21c0d679d4efc7 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
09:13:46.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.840132 24049 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 2ea1d9e208a44e998b21c0d679d4efc7 [term 0 FOLLOWER]: Advancing to term 1
09:13:46.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.846136 24049 raft_consensus.cc:515] T 00000000000000000000000000000000 P 2ea1d9e208a44e998b21c0d679d4efc7 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "771c7c9f49674c838e35b35ae30ef674" member_type: VOTER last_known_addr { host: "127.23.63.126" port: 45639 } } peers { permanent_uuid: "ed9a912ea092472eae7d7af380068f95" member_type: VOTER last_known_addr { host: "127.23.63.125" port: 34901 } } peers { permanent_uuid: "2ea1d9e208a44e998b21c0d679d4efc7" member_type: VOTER last_known_addr { host: "127.23.63.124" port: 43733 } }
09:13:46.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.849480 23935 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2ea1d9e208a44e998b21c0d679d4efc7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ed9a912ea092472eae7d7af380068f95"
09:13:46.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.852169 23935 raft_consensus.cc:3060] T 00000000000000000000000000000000 P ed9a912ea092472eae7d7af380068f95 [term 0 FOLLOWER]: Advancing to term 1
09:13:46.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:46.855306 23914 tablet.cc:2378] T 00000000000000000000000000000000 P ed9a912ea092472eae7d7af380068f95: Can't schedule compaction. Clean time has not been advanced past its initial value.
09:13:46.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.858079 24049 leader_election.cc:290] T 00000000000000000000000000000000 P 2ea1d9e208a44e998b21c0d679d4efc7 [CANDIDATE]: Term 1 election: Requested vote from peers 771c7c9f49674c838e35b35ae30ef674 (127.23.63.126:45639), ed9a912ea092472eae7d7af380068f95 (127.23.63.125:34901)
09:13:46.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.862984 23935 raft_consensus.cc:2468] T 00000000000000000000000000000000 P ed9a912ea092472eae7d7af380068f95 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2ea1d9e208a44e998b21c0d679d4efc7 in term 1.
09:13:46.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.864312 23989 leader_election.cc:304] T 00000000000000000000000000000000 P 2ea1d9e208a44e998b21c0d679d4efc7 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2ea1d9e208a44e998b21c0d679d4efc7, ed9a912ea092472eae7d7af380068f95; no voters:
09:13:46.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.865185 24049 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 2ea1d9e208a44e998b21c0d679d4efc7 [term 1 FOLLOWER]: Leader election won for term 1
09:13:46.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.874166 24049 raft_consensus.cc:697] T 00000000000000000000000000000000 P 2ea1d9e208a44e998b21c0d679d4efc7 [term 1 LEADER]: Becoming Leader. State: Replica: 2ea1d9e208a44e998b21c0d679d4efc7, State: Running, Role: LEADER
09:13:46.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.875615 24049 consensus_queue.cc:237] T 00000000000000000000000000000000 P 2ea1d9e208a44e998b21c0d679d4efc7 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "771c7c9f49674c838e35b35ae30ef674" member_type: VOTER last_known_addr { host: "127.23.63.126" port: 45639 } } peers { permanent_uuid: "ed9a912ea092472eae7d7af380068f95" member_type: VOTER last_known_addr { host: "127.23.63.125" port: 34901 } } peers { permanent_uuid: "2ea1d9e208a44e998b21c0d679d4efc7" member_type: VOTER last_known_addr { host: "127.23.63.124" port: 43733 } }
09:13:46.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.895063 24070 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2ea1d9e208a44e998b21c0d679d4efc7 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 2ea1d9e208a44e998b21c0d679d4efc7. Latest consensus state: current_term: 1 leader_uuid: "2ea1d9e208a44e998b21c0d679d4efc7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "771c7c9f49674c838e35b35ae30ef674" member_type: VOTER last_known_addr { host: "127.23.63.126" port: 45639 } } peers { permanent_uuid: "ed9a912ea092472eae7d7af380068f95" member_type: VOTER last_known_addr { host: "127.23.63.125" port: 34901 } } peers { permanent_uuid: "2ea1d9e208a44e998b21c0d679d4efc7" member_type: VOTER last_known_addr { host: "127.23.63.124" port: 43733 } } }
09:13:46.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.895658 24070 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2ea1d9e208a44e998b21c0d679d4efc7 [sys.catalog]: This master's current role is: LEADER
09:13:46.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.904371 24079 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
09:13:46.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.911405 24079 catalog_manager.cc:1494] Initializing Kudu cluster ID...
09:13:46.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.892652 23887 master_runner.cc:186] Error getting master registration for 127.23.63.126:45639: OK, instance_id { permanent_uuid: "771c7c9f49674c838e35b35ae30ef674" instance_seqno: 1761556422481599 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
09:13:46.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.920023 23887 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
09:13:46.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:46.920603 24077 catalog_manager.cc:1568] T 00000000000000000000000000000000 P ed9a912ea092472eae7d7af380068f95: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
09:13:46.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:46.920885 24077 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
09:13:46.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.939285 23935 raft_consensus.cc:1275] T 00000000000000000000000000000000 P ed9a912ea092472eae7d7af380068f95 [term 1 FOLLOWER]: Refusing update from remote peer 2ea1d9e208a44e998b21c0d679d4efc7: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
09:13:46.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.940747 24049 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 2ea1d9e208a44e998b21c0d679d4efc7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ed9a912ea092472eae7d7af380068f95" member_type: VOTER last_known_addr { host: "127.23.63.125" port: 34901 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:13:46.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.960321 24064 sys_catalog.cc:455] T 00000000000000000000000000000000 P ed9a912ea092472eae7d7af380068f95 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 2ea1d9e208a44e998b21c0d679d4efc7. Latest consensus state: current_term: 1 leader_uuid: "2ea1d9e208a44e998b21c0d679d4efc7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "771c7c9f49674c838e35b35ae30ef674" member_type: VOTER last_known_addr { host: "127.23.63.126" port: 45639 } } peers { permanent_uuid: "ed9a912ea092472eae7d7af380068f95" member_type: VOTER last_known_addr { host: "127.23.63.125" port: 34901 } } peers { permanent_uuid: "2ea1d9e208a44e998b21c0d679d4efc7" member_type: VOTER last_known_addr { host: "127.23.63.124" port: 43733 } } }
09:13:46.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.960889 24064 sys_catalog.cc:458] T 00000000000000000000000000000000 P ed9a912ea092472eae7d7af380068f95 [sys.catalog]: This master's current role is: FOLLOWER
09:13:46.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.963816 24080 mvcc.cc:204] Tried to move back new op lower bound from 7215335124721676288 to 7215335124500520960. Current Snapshot: MvccSnapshot[applied={T|T < 7215335124721676288}]
09:13:46.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.971406 24049 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2ea1d9e208a44e998b21c0d679d4efc7 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "2ea1d9e208a44e998b21c0d679d4efc7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "771c7c9f49674c838e35b35ae30ef674" member_type: VOTER last_known_addr { host: "127.23.63.126" port: 45639 } } peers { permanent_uuid: "ed9a912ea092472eae7d7af380068f95" member_type: VOTER last_known_addr { host: "127.23.63.125" port: 34901 } } peers { permanent_uuid: "2ea1d9e208a44e998b21c0d679d4efc7" member_type: VOTER last_known_addr { host: "127.23.63.124" port: 43733 } } }
09:13:46.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.972034 24049 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2ea1d9e208a44e998b21c0d679d4efc7 [sys.catalog]: This master's current role is: LEADER
09:13:46.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.974733 24064 sys_catalog.cc:455] T 00000000000000000000000000000000 P ed9a912ea092472eae7d7af380068f95 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "2ea1d9e208a44e998b21c0d679d4efc7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "771c7c9f49674c838e35b35ae30ef674" member_type: VOTER last_known_addr { host: "127.23.63.126" port: 45639 } } peers { permanent_uuid: "ed9a912ea092472eae7d7af380068f95" member_type: VOTER last_known_addr { host: "127.23.63.125" port: 34901 } } peers { permanent_uuid: "2ea1d9e208a44e998b21c0d679d4efc7" member_type: VOTER last_known_addr { host: "127.23.63.124" port: 43733 } } }
09:13:46.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.975282 24064 sys_catalog.cc:458] T 00000000000000000000000000000000 P ed9a912ea092472eae7d7af380068f95 [sys.catalog]: This master's current role is: FOLLOWER
09:13:46.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.979691 24079 catalog_manager.cc:1357] Generated new cluster ID: 2f58e2b8eb604fe7a81169504492fa7f
09:13:46.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:46.979974 24079 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
09:13:47.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:47.015544 24079 catalog_manager.cc:1380] Generated new certificate authority record
09:13:47.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:47.019169 24079 catalog_manager.cc:1514] Loading token signing keys...
09:13:47.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:47.040067 24079 catalog_manager.cc:6022] T 00000000000000000000000000000000 P 2ea1d9e208a44e998b21c0d679d4efc7: Generated new TSK 0
09:13:47.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:47.040961 24079 catalog_manager.cc:1524] Initializing in-progress tserver states...
09:13:47.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:47.061803 23805 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu
09:13:47.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu
09:13:47.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-0/wal
09:13:47.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-0/data
09:13:47.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:13:47.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:13:47.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:13:47.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:13:47.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:13:47.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:13:47.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-0/logs
09:13:47.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-0/data/info.pb
09:13:47.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:13:47.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:13:47.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:13:47.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:13:47.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:13:47.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:13:47.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:13:47.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:13:47.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
09:13:47.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:13:47.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.63.65:0
09:13:47.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.63.65
09:13:47.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.63.65
09:13:47.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:13:47.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.63.126:45639,127.23.63.125:34901,127.23.63.124:43733
09:13:47.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.63.84:45099
09:13:47.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:13:47.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:13:47.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
09:13:47.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:47.187296 23885 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.63.124" port: 43733 } attempt: 7
09:13:47.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:47.214670 23885 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 771c7c9f49674c838e35b35ae30ef674: Bootstrap starting.
09:13:47.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:47.233263 23885 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 771c7c9f49674c838e35b35ae30ef674: Neither blocks nor log segments found. Creating new log.
09:13:47.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:47.235318 23885 log.cc:826] T 00000000000000000000000000000000 P 771c7c9f49674c838e35b35ae30ef674: Log is configured to *not* fsync() on all Append() calls
09:13:47.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:47.240530 23885 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 771c7c9f49674c838e35b35ae30ef674: No bootstrap required, opened a new log
09:13:47.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:47.297163 23885 raft_consensus.cc:359] T 00000000000000000000000000000000 P 771c7c9f49674c838e35b35ae30ef674 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "771c7c9f49674c838e35b35ae30ef674" member_type: VOTER last_known_addr { host: "127.23.63.126" port: 45639 } } peers { permanent_uuid: "ed9a912ea092472eae7d7af380068f95" member_type: VOTER last_known_addr { host: "127.23.63.125" port: 34901 } } peers { permanent_uuid: "2ea1d9e208a44e998b21c0d679d4efc7" member_type: VOTER last_known_addr { host: "127.23.63.124" port: 43733 } }
09:13:47.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:47.298923 23885 raft_consensus.cc:385] T 00000000000000000000000000000000 P 771c7c9f49674c838e35b35ae30ef674 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:13:47.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:47.299476 23885 raft_consensus.cc:740] T 00000000000000000000000000000000 P 771c7c9f49674c838e35b35ae30ef674 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 771c7c9f49674c838e35b35ae30ef674, State: Initialized, Role: FOLLOWER
09:13:47.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:47.301358 23885 consensus_queue.cc:260] T 00000000000000000000000000000000 P 771c7c9f49674c838e35b35ae30ef674 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "771c7c9f49674c838e35b35ae30ef674" member_type: VOTER last_known_addr { host: "127.23.63.126" port: 45639 } } peers { permanent_uuid: "ed9a912ea092472eae7d7af380068f95" member_type: VOTER last_known_addr { host: "127.23.63.125" port: 34901 } } peers { permanent_uuid: "2ea1d9e208a44e998b21c0d679d4efc7" member_type: VOTER last_known_addr { host: "127.23.63.124" port: 43733 } }
09:13:47.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:47.332518 24088 sys_catalog.cc:455] T 00000000000000000000000000000000 P 771c7c9f49674c838e35b35ae30ef674 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "771c7c9f49674c838e35b35ae30ef674" member_type: VOTER last_known_addr { host: "127.23.63.126" port: 45639 } } peers { permanent_uuid: "ed9a912ea092472eae7d7af380068f95" member_type: VOTER last_known_addr { host: "127.23.63.125" port: 34901 } } peers { permanent_uuid: "2ea1d9e208a44e998b21c0d679d4efc7" member_type: VOTER last_known_addr { host: "127.23.63.124" port: 43733 } } }
09:13:47.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:47.335916 24088 sys_catalog.cc:458] T 00000000000000000000000000000000 P 771c7c9f49674c838e35b35ae30ef674 [sys.catalog]: This master's current role is: FOLLOWER
09:13:47.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:47.339877 23859 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2ea1d9e208a44e998b21c0d679d4efc7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "771c7c9f49674c838e35b35ae30ef674"
09:13:47.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:47.340592 23860 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2ea1d9e208a44e998b21c0d679d4efc7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "771c7c9f49674c838e35b35ae30ef674" is_pre_election: true
09:13:47.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:47.369361 23885 sys_catalog.cc:565] T 00000000000000000000000000000000 P 771c7c9f49674c838e35b35ae30ef674 [sys.catalog]: configured and running, proceeding with master startup.
09:13:47.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:47.380472 23858 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 771c7c9f49674c838e35b35ae30ef674 [term 0 FOLLOWER]: Advancing to term 1
09:13:47.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:47.392200 23858 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 771c7c9f49674c838e35b35ae30ef674 [term 1 FOLLOWER]: Refusing update from remote peer 2ea1d9e208a44e998b21c0d679d4efc7: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
09:13:47.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:47.394771 24070 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 2ea1d9e208a44e998b21c0d679d4efc7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "771c7c9f49674c838e35b35ae30ef674" member_type: VOTER last_known_addr { host: "127.23.63.126" port: 45639 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:13:47.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:47.446591 24088 sys_catalog.cc:455] T 00000000000000000000000000000000 P 771c7c9f49674c838e35b35ae30ef674 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 2ea1d9e208a44e998b21c0d679d4efc7. Latest consensus state: current_term: 1 leader_uuid: "2ea1d9e208a44e998b21c0d679d4efc7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "771c7c9f49674c838e35b35ae30ef674" member_type: VOTER last_known_addr { host: "127.23.63.126" port: 45639 } } peers { permanent_uuid: "ed9a912ea092472eae7d7af380068f95" member_type: VOTER last_known_addr { host: "127.23.63.125" port: 34901 } } peers { permanent_uuid: "2ea1d9e208a44e998b21c0d679d4efc7" member_type: VOTER last_known_addr { host: "127.23.63.124" port: 43733 } } }
09:13:47.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:47.447157 24088 sys_catalog.cc:458] T 00000000000000000000000000000000 P 771c7c9f49674c838e35b35ae30ef674 [sys.catalog]: This master's current role is: FOLLOWER
09:13:47.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:47.446700 24097 sys_catalog.cc:455] T 00000000000000000000000000000000 P 771c7c9f49674c838e35b35ae30ef674 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "2ea1d9e208a44e998b21c0d679d4efc7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "771c7c9f49674c838e35b35ae30ef674" member_type: VOTER last_known_addr { host: "127.23.63.126" port: 45639 } } peers { permanent_uuid: "ed9a912ea092472eae7d7af380068f95" member_type: VOTER last_known_addr { host: "127.23.63.125" port: 34901 } } peers { permanent_uuid: "2ea1d9e208a44e998b21c0d679d4efc7" member_type: VOTER last_known_addr { host: "127.23.63.124" port: 43733 } } }
09:13:47.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:47.447727 24097 sys_catalog.cc:458] T 00000000000000000000000000000000 P 771c7c9f49674c838e35b35ae30ef674 [sys.catalog]: This master's current role is: FOLLOWER
09:13:47.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:47.461043 24107 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
09:13:47.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:47.517869 24085 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:13:47.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:47.518422 24085 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:13:47.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:47.519100 24085 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:13:47.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:47.568312 24085 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:13:47.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:47.569512 24085 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.63.65
09:13:47.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:47.623677 24085 tablet_server_runner.cc:78] Tablet server non-default flags:
09:13:47.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:13:47.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.63.84:45099
09:13:47.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:13:47.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:13:47.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-0/data
09:13:47.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-0/wal
09:13:47.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:13:47.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:13:47.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.63.65:0
09:13:47.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:13:47.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:13:47.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:13:47.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-0/data/info.pb
09:13:47.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.63.65
09:13:47.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:13:47.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.63.126:45639,127.23.63.125:34901,127.23.63.124:43733
09:13:47.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:13:47.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:13:47.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:13:47.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:13:47.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:13:47.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.63.65
09:13:47.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-0/logs
09:13:47.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:13:47.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:13:47.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:13:47.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
09:13:47.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:13:47.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d0cb013a5aad754f2ed06c25f5ec3d8a1873622
09:13:47.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:13:47.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Oct 2025 08:43:21 UTC on 5fd53c4cbb9d
09:13:47.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8741
09:13:47.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:13:47.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:47.625068 24085 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:13:47.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:47.626844 24085 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:13:47.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:47.644829 24110 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:47.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:47.830507 24114 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2ea1d9e208a44e998b21c0d679d4efc7 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "2ea1d9e208a44e998b21c0d679d4efc7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "771c7c9f49674c838e35b35ae30ef674" member_type: VOTER last_known_addr { host: "127.23.63.126" port: 45639 } } peers { permanent_uuid: "ed9a912ea092472eae7d7af380068f95" member_type: VOTER last_known_addr { host: "127.23.63.125" port: 34901 } } peers { permanent_uuid: "2ea1d9e208a44e998b21c0d679d4efc7" member_type: VOTER last_known_addr { host: "127.23.63.124" port: 43733 } } }
09:13:47.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:47.834870 24114 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2ea1d9e208a44e998b21c0d679d4efc7 [sys.catalog]: This master's current role is: LEADER
09:13:47.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:47.925767 24077 catalog_manager.cc:1269] Loaded cluster ID: 2f58e2b8eb604fe7a81169504492fa7f
09:13:47.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:47.926165 24077 catalog_manager.cc:1562] T 00000000000000000000000000000000 P ed9a912ea092472eae7d7af380068f95: loading cluster ID for follower catalog manager: success
09:13:47.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:47.932403 24077 catalog_manager.cc:1584] T 00000000000000000000000000000000 P ed9a912ea092472eae7d7af380068f95: acquiring CA information for follower catalog manager: success
09:13:47.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:47.941345 24077 catalog_manager.cc:1612] T 00000000000000000000000000000000 P ed9a912ea092472eae7d7af380068f95: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:13:48.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:48.475595 24107 catalog_manager.cc:1269] Loaded cluster ID: 2f58e2b8eb604fe7a81169504492fa7f
09:13:48.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:48.475822 24107 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 771c7c9f49674c838e35b35ae30ef674: loading cluster ID for follower catalog manager: success
09:13:48.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:48.497545 24107 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 771c7c9f49674c838e35b35ae30ef674: acquiring CA information for follower catalog manager: success
09:13:48.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:48.509608 24107 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 771c7c9f49674c838e35b35ae30ef674: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:13:49.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:49.044868 24109 debug-util.cc:398] Leaking SignalData structure 0x7b0800037500 after lost signal to thread 24085
09:13:49.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:49.114105 24085 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.471s user 0.427s sys 0.730s
09:13:49.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:49.114588 24085 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.472s user 0.427s sys 0.730s
09:13:49.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:47.645773 24111 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:49.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:49.131769 24085 server_base.cc:1047] running on GCE node
09:13:49.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:49.135941 24119 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:49.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:49.137137 24085 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:13:49.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:49.147070 24085 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:13:49.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:49.148535 24085 hybrid_clock.cc:648] HybridClock initialized: now 1761556429148496 us; error 33 us; skew 500 ppm
09:13:49.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:49.149586 24085 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:13:49.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:49.172343 24085 webserver.cc:492] Webserver started at http://127.23.63.65:34053/ using document root <none> and password file <none>
09:13:49.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:49.173537 24085 fs_manager.cc:362] Metadata directory not provided
09:13:49.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:49.173897 24085 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:13:49.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:49.174470 24085 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
09:13:49.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:49.180212 24085 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-0/data/instance:
09:13:49.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b7459cba87c740e29ddc1af473c773a2"
09:13:49.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-27 09:13:49 on dist-test-slave-n41w"
09:13:49.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:49.181667 24085 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-0/wal/instance:
09:13:49.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b7459cba87c740e29ddc1af473c773a2"
09:13:49.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-27 09:13:49 on dist-test-slave-n41w"
09:13:49.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:49.195617 24085 fs_manager.cc:696] Time spent creating directory manager: real 0.013s user 0.012s sys 0.000s
09:13:49.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:49.202534 24124 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:13:49.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:49.203652 24085 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
09:13:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:49.204079 24085 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-0/data,/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-0/wal
09:13:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b7459cba87c740e29ddc1af473c773a2"
09:13:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-27 09:13:49 on dist-test-slave-n41w"
09:13:49.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:49.204552 24085 fs_report.cc:389] FS layout report
09:13:49.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:13:49.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-0/wal
09:13:49.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-0/wal
09:13:49.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-0/data/data
09:13:49.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:13:49.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:13:49.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:13:49.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:13:49.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:13:49.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:13:49.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:13:49.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:13:49.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:13:49.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:13:49.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:13:49.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:49.281955 24085 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:13:49.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:49.283243 24085 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:13:49.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:49.283663 24085 kserver.cc:163] Server-wide thread pool size limit: 3276
09:13:49.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:49.286288 24085 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
09:13:49.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:49.290558 24085 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
09:13:49.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:49.290733 24085 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
09:13:49.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:49.291004 24085 ts_tablet_manager.cc:616] Registered 0 tablets
09:13:49.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:49.291144 24085 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
09:13:49.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:49.480155 24085 rpc_server.cc:307] RPC server started. Bound to: 127.23.63.65:37885
09:13:49.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:49.480255 24236 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.63.65:37885 every 8 connection(s)
09:13:49.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:49.482803 24085 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-0/data/info.pb
09:13:49.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:49.493310 23805 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu as pid 24085
09:13:49.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:49.493813 23805 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-0/wal/instance
09:13:49.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:49.523551 23805 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu
09:13:49.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu
09:13:49.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-1/wal
09:13:49.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-1/data
09:13:49.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:13:49.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:13:49.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:13:49.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:13:49.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:13:49.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:13:49.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-1/logs
09:13:49.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-1/data/info.pb
09:13:49.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:13:49.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:13:49.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:13:49.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:13:49.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:13:49.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:13:49.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:13:49.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:13:49.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
09:13:49.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:13:49.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.63.66:0
09:13:49.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.63.66
09:13:49.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.63.66
09:13:49.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:13:49.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.63.126:45639,127.23.63.125:34901,127.23.63.124:43733
09:13:49.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.63.84:45099
09:13:49.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:13:49.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:13:49.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
09:13:49.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:49.608310 24242 heartbeater.cc:344] Connected to a master server at 127.23.63.126:45639
09:13:49.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:49.608753 24242 heartbeater.cc:461] Registering TS with master...
09:13:49.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:49.609383 24238 heartbeater.cc:344] Connected to a master server at 127.23.63.125:34901
09:13:49.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:49.609690 24238 heartbeater.cc:461] Registering TS with master...
09:13:49.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:49.609982 24242 heartbeater.cc:507] Master 127.23.63.126:45639 requested a full tablet report, sending...
09:13:49.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:49.610613 24238 heartbeater.cc:507] Master 127.23.63.125:34901 requested a full tablet report, sending...
09:13:49.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:49.613781 23925 ts_manager.cc:194] Registered new tserver with Master: b7459cba87c740e29ddc1af473c773a2 (127.23.63.65:37885)
09:13:49.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:49.614913 24237 heartbeater.cc:344] Connected to a master server at 127.23.63.124:43733
09:13:49.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:49.615152 24237 heartbeater.cc:461] Registering TS with master...
09:13:49.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:49.615792 24237 heartbeater.cc:507] Master 127.23.63.124:43733 requested a full tablet report, sending...
09:13:49.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:49.617801 23850 ts_manager.cc:194] Registered new tserver with Master: b7459cba87c740e29ddc1af473c773a2 (127.23.63.65:37885)
09:13:49.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:49.619343 24005 ts_manager.cc:194] Registered new tserver with Master: b7459cba87c740e29ddc1af473c773a2 (127.23.63.65:37885)
09:13:49.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:49.620939 24005 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.23.63.65:37279
09:13:49.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:49.871628 24241 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:13:49.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:49.872092 24241 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:13:49.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:49.872534 24241 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:13:49.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:49.899394 24241 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:13:49.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:49.900115 24241 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.63.66
09:13:49.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:49.932025 24241 tablet_server_runner.cc:78] Tablet server non-default flags:
09:13:49.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:13:49.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.63.84:45099
09:13:49.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:13:49.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:13:49.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-1/data
09:13:49.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-1/wal
09:13:49.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:13:49.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:13:49.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.63.66:0
09:13:49.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:13:49.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:13:49.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:13:49.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-1/data/info.pb
09:13:49.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.63.66
09:13:49.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:13:49.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.63.126:45639,127.23.63.125:34901,127.23.63.124:43733
09:13:49.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:13:49.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:13:49.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:13:49.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:13:49.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:13:49.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.63.66
09:13:49.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-1/logs
09:13:49.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:13:49.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:13:49.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:13:49.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
09:13:49.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:13:49.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d0cb013a5aad754f2ed06c25f5ec3d8a1873622
09:13:49.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:13:49.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Oct 2025 08:43:21 UTC on 5fd53c4cbb9d
09:13:49.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8741
09:13:49.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:13:49.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:49.933385 24241 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:13:49.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:49.935199 24241 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:13:49.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:49.949007 24253 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:50.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:50.623992 24237 heartbeater.cc:499] Master 127.23.63.124:43733 was elected leader, sending a full tablet report...
09:13:51.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:51.353377 24252 debug-util.cc:398] Leaking SignalData structure 0x7b0800037500 after lost signal to thread 24241
09:13:51.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:51.585685 24252 kernel_stack_watchdog.cc:198] Thread 24241 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 402ms:
09:13:51.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
09:13:51.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
09:13:51.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
09:13:51.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
09:13:51.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:49.950402 24254 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:51.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:51.586738 24241 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.638s user 0.520s sys 0.925s
09:13:51.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:51.587037 24241 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.638s user 0.520s sys 0.925s
09:13:51.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:51.587272 24255 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1636 milliseconds
09:13:51.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:51.588261 24256 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:51.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:51.588341 24241 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:13:51.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:51.592195 24241 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:13:51.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:51.598650 24241 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:13:51.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:51.600013 24241 hybrid_clock.cc:648] HybridClock initialized: now 1761556431599986 us; error 29 us; skew 500 ppm
09:13:51.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:51.600741 24241 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:13:51.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:51.606223 24241 webserver.cc:492] Webserver started at http://127.23.63.66:40069/ using document root <none> and password file <none>
09:13:51.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:51.607009 24241 fs_manager.cc:362] Metadata directory not provided
09:13:51.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:51.607185 24241 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:13:51.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:51.607635 24241 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
09:13:51.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:51.611340 24241 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-1/data/instance:
09:13:51.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5d5c9e655e0949c8af3b5d015060b25d"
09:13:51.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-27 09:13:51 on dist-test-slave-n41w"
09:13:51.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:51.612493 24241 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-1/wal/instance:
09:13:51.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5d5c9e655e0949c8af3b5d015060b25d"
09:13:51.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-27 09:13:51 on dist-test-slave-n41w"
09:13:51.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:51.618830 24241 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.006s sys 0.000s
09:13:51.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:51.624727 24267 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:13:51.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:51.625948 24241 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
09:13:51.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:51.626251 24241 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-1/data,/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-1/wal
09:13:51.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5d5c9e655e0949c8af3b5d015060b25d"
09:13:51.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-27 09:13:51 on dist-test-slave-n41w"
09:13:51.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:51.626585 24241 fs_report.cc:389] FS layout report
09:13:51.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:13:51.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-1/wal
09:13:51.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-1/wal
09:13:51.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-1/data/data
09:13:51.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:13:51.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:13:51.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:13:51.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:13:51.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:13:51.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:13:51.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:13:51.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:13:51.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:13:51.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:13:51.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:13:51.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:51.688226 24241 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:13:51.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:51.689576 24241 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:13:51.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:51.689970 24241 kserver.cc:163] Server-wide thread pool size limit: 3276
09:13:51.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:51.692248 24241 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
09:13:51.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:51.695807 24241 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
09:13:51.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:51.696064 24241 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
09:13:51.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:51.696235 24241 ts_tablet_manager.cc:616] Registered 0 tablets
09:13:51.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:51.696367 24241 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
09:13:51.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:51.811116 24241 rpc_server.cc:307] RPC server started. Bound to: 127.23.63.66:36525
09:13:51.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:51.811229 24379 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.63.66:36525 every 8 connection(s)
09:13:51.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:51.813223 24241 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-1/data/info.pb
09:13:51.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:51.823462 23805 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu as pid 24241
09:13:51.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:51.824038 23805 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-1/wal/instance
09:13:51.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:51.836558 23805 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu
09:13:51.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu
09:13:51.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-2/wal
09:13:51.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-2/data
09:13:51.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:13:51.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:13:51.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:13:51.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:13:51.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:13:51.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:13:51.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-2/logs
09:13:51.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-2/data/info.pb
09:13:51.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:13:51.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:13:51.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:13:51.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:13:51.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:13:51.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:13:51.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:13:51.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:13:51.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
09:13:51.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:13:51.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.63.67:0
09:13:51.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.63.67
09:13:51.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.63.67
09:13:51.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:13:51.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.63.126:45639,127.23.63.125:34901,127.23.63.124:43733
09:13:51.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.63.84:45099
09:13:51.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:13:51.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:13:51.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
09:13:51.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:51.845943 24380 heartbeater.cc:344] Connected to a master server at 127.23.63.124:43733
09:13:51.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:51.846386 24381 heartbeater.cc:344] Connected to a master server at 127.23.63.125:34901
09:13:51.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:51.846411 24380 heartbeater.cc:461] Registering TS with master...
09:13:51.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:51.846693 24381 heartbeater.cc:461] Registering TS with master...
09:13:51.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:51.847579 24381 heartbeater.cc:507] Master 127.23.63.125:34901 requested a full tablet report, sending...
09:13:51.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:51.847664 24380 heartbeater.cc:507] Master 127.23.63.124:43733 requested a full tablet report, sending...
09:13:51.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:51.850025 24005 ts_manager.cc:194] Registered new tserver with Master: 5d5c9e655e0949c8af3b5d015060b25d (127.23.63.66:36525)
09:13:51.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:51.850415 23925 ts_manager.cc:194] Registered new tserver with Master: 5d5c9e655e0949c8af3b5d015060b25d (127.23.63.66:36525)
09:13:51.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:51.851564 24005 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.23.63.66:60165
09:13:51.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:51.852058 24382 heartbeater.cc:344] Connected to a master server at 127.23.63.126:45639
09:13:51.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:51.852336 24382 heartbeater.cc:461] Registering TS with master...
09:13:51.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:51.852998 24382 heartbeater.cc:507] Master 127.23.63.126:45639 requested a full tablet report, sending...
09:13:51.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:51.854640 23850 ts_manager.cc:194] Registered new tserver with Master: 5d5c9e655e0949c8af3b5d015060b25d (127.23.63.66:36525)
09:13:52.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:52.110110 24390 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:13:52.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:52.110489 24390 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:13:52.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:52.110903 24390 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:13:52.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:52.139201 24390 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:13:52.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:52.139961 24390 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.63.67
09:13:52.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:52.170931 24390 tablet_server_runner.cc:78] Tablet server non-default flags:
09:13:52.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:13:52.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.63.84:45099
09:13:52.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:13:52.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:13:52.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-2/data
09:13:52.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-2/wal
09:13:52.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:13:52.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:13:52.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.63.67:0
09:13:52.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:13:52.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:13:52.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:13:52.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-2/data/info.pb
09:13:52.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.63.67
09:13:52.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:13:52.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.63.126:45639,127.23.63.125:34901,127.23.63.124:43733
09:13:52.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:13:52.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:13:52.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:13:52.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:13:52.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:13:52.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.63.67
09:13:52.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-2/logs
09:13:52.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:13:52.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:13:52.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:13:52.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
09:13:52.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:13:52.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d0cb013a5aad754f2ed06c25f5ec3d8a1873622
09:13:52.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:13:52.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Oct 2025 08:43:21 UTC on 5fd53c4cbb9d
09:13:52.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8741
09:13:52.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:13:52.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:52.172065 24390 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:13:52.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:52.173419 24390 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:13:52.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:52.184404 24396 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:52.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:52.854570 24380 heartbeater.cc:499] Master 127.23.63.124:43733 was elected leader, sending a full tablet report...
09:13:53.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:53.695295 24390 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.511s user 0.460s sys 1.038s
09:13:53.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:53.587337 24395 debug-util.cc:398] Leaking SignalData structure 0x7b0800037500 after lost signal to thread 24390
09:13:53.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:52.184880 24397 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:53.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:53.695704 24390 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.512s user 0.460s sys 1.038s
09:13:53.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:53.696700 24398 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1510 milliseconds
09:13:53.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:53.697121 24390 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:13:53.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:53.697214 24399 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:53.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:53.700330 24390 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:13:53.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:53.702160 24390 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:13:53.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:53.703459 24390 hybrid_clock.cc:648] HybridClock initialized: now 1761556433703407 us; error 46 us; skew 500 ppm
09:13:53.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:53.704208 24390 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:13:53.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:53.709373 24390 webserver.cc:492] Webserver started at http://127.23.63.67:42119/ using document root <none> and password file <none>
09:13:53.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:53.710177 24390 fs_manager.cc:362] Metadata directory not provided
09:13:53.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:53.710361 24390 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:13:53.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:53.710757 24390 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
09:13:53.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:53.714656 24390 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-2/data/instance:
09:13:53.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d3a2a8cd02384ab58e731542d7dc06c6"
09:13:53.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-27 09:13:53 on dist-test-slave-n41w"
09:13:53.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:53.715622 24390 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-2/wal/instance:
09:13:53.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d3a2a8cd02384ab58e731542d7dc06c6"
09:13:53.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-27 09:13:53 on dist-test-slave-n41w"
09:13:53.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:53.721729 24390 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.005s sys 0.000s
09:13:53.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:53.726497 24407 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:13:53.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:53.727548 24390 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
09:13:53.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:53.727897 24390 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-2/data,/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-2/wal
09:13:53.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d3a2a8cd02384ab58e731542d7dc06c6"
09:13:53.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-27 09:13:53 on dist-test-slave-n41w"
09:13:53.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:53.728168 24390 fs_report.cc:389] FS layout report
09:13:53.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:13:53.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-2/wal
09:13:53.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-2/wal
09:13:53.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-2/data/data
09:13:53.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:13:53.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:13:53.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:13:53.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:13:53.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:13:53.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:13:53.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:13:53.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:13:53.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:13:53.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:13:53.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:13:53.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:53.778925 24390 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:13:53.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:53.780165 24390 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:13:53.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:53.780529 24390 kserver.cc:163] Server-wide thread pool size limit: 3276
09:13:53.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:53.782590 24390 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
09:13:53.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:53.786280 24390 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
09:13:53.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:53.786458 24390 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
09:13:53.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:53.786708 24390 ts_tablet_manager.cc:616] Registered 0 tablets
09:13:53.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:53.786862 24390 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
09:13:53.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:53.911254 24390 rpc_server.cc:307] RPC server started. Bound to: 127.23.63.67:42983
09:13:53.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:53.911315 24519 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.63.67:42983 every 8 connection(s)
09:13:53.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:53.913371 24390 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-2/data/info.pb
09:13:53.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:53.916361 23805 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu as pid 24390
09:13:53.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:53.916740 23805 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster5159703600514210066/ts-2/wal/instance
09:13:53.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:53.953806 24521 heartbeater.cc:344] Connected to a master server at 127.23.63.125:34901
09:13:53.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:53.954237 24521 heartbeater.cc:461] Registering TS with master...
09:13:53.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:53.954509 24522 heartbeater.cc:344] Connected to a master server at 127.23.63.126:45639
09:13:53.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:53.954807 24520 heartbeater.cc:344] Connected to a master server at 127.23.63.124:43733
09:13:53.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:53.955119 24520 heartbeater.cc:461] Registering TS with master...
09:13:53.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:53.955348 24521 heartbeater.cc:507] Master 127.23.63.125:34901 requested a full tablet report, sending...
09:13:53.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:53.954839 24522 heartbeater.cc:461] Registering TS with master...
09:13:53.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:53.956089 24522 heartbeater.cc:507] Master 127.23.63.126:45639 requested a full tablet report, sending...
09:13:53.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:53.956598 24520 heartbeater.cc:507] Master 127.23.63.124:43733 requested a full tablet report, sending...
09:13:53.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:53.957994 23850 ts_manager.cc:194] Registered new tserver with Master: d3a2a8cd02384ab58e731542d7dc06c6 (127.23.63.67:42983)
09:13:53.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:53.957989 23925 ts_manager.cc:194] Registered new tserver with Master: d3a2a8cd02384ab58e731542d7dc06c6 (127.23.63.67:42983)
09:13:53.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:53.959658 24005 ts_manager.cc:194] Registered new tserver with Master: d3a2a8cd02384ab58e731542d7dc06c6 (127.23.63.67:42983)
09:13:53.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:53.960772 24005 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.23.63.67:43615
09:13:53.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:53.971267 23805 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
09:13:53.981 [DEBUG - main] (MiniKuduCluster.java:195) Response:
09:13:53.981 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
09:13:53.982 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.23.63.126"
port: 45639
}
bound_http_address {
host: "127.23.63.126"
port: 39623
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.23.63.125"
port: 34901
}
bound_http_address {
host: "127.23.63.125"
port: 42915
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.23.63.124"
port: 43733
}
bound_http_address {
host: "127.23.63.124"
port: 42007
}
}
}
09:13:53.982 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
09:13:53.983 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.23.63.65"
port: 37885
}
bound_http_address {
host: "127.23.63.65"
port: 34053
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.23.63.66"
port: 36525
}
bound_http_address {
host: "127.23.63.66"
port: 40069
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.23.63.67"
port: 42983
}
bound_http_address {
host: "127.23.63.67"
port: 42119
}
}
}
09:13:53.983 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
09:13:53.984 [DEBUG - main] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster6389304023375265688/krb5kdc/krb5cc
09:13:53.984 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
09:13:53.986 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskWfs5gY/test-tmp
09:13:53.986 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
09:13:53.986 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskWfs5gY/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
09:13:53.988 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
09:13:54.263 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:13:54.263 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
09:13:54.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:54.265661 24531 mini_oidc.cc:150] Starting JWKS server
09:13:54.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:54.268349 24531 webserver.cc:492] Webserver started at http://127.0.0.1:38985/ using document root <none> and password file <none>
09:13:54.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:54.268687 24531 mini_oidc.cc:186] Starting OIDC Discovery server
09:13:54.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:54.270143 24531 webserver.cc:492] Webserver started at http://127.0.0.1:44015/ using document root <none> and password file <none>
09:13:54.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
09:13:54.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/principal' for realm 'KRBTEST.COM',
09:13:54.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
09:13:54.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 27 09:13:54 dist-test-slave-n41w krb5kdc[24542](info): setting up network...
09:13:54.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
09:13:54.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 27 09:13:54 dist-test-slave-n41w krb5kdc[24542](info): set up 2 sockets
09:13:54.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 27 09:13:54 dist-test-slave-n41w krb5kdc[24542](info): commencing operation
09:13:54.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
09:13:54.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:54.962946 24520 heartbeater.cc:499] Master 127.23.63.124:43733 was elected leader, sending a full tablet report...
09:13:56.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:56.083295 24531 mini_kdc.cc:121] Time spent starting KDC: real 1.782s user 0.001s sys 0.013s
09:13:56.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
09:13:56.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
09:13:56.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
09:13:56.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
09:13:56.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
09:13:56.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
09:13:56.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
09:13:56.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
09:13:56.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
09:13:56.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
09:13:56.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/test-user.keytab.
09:13:56.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/test-user.keytab.
09:13:56.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 27 09:13:56 dist-test-slave-n41w krb5kdc[24542](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761556436, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:13:56.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
09:13:56.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-27T09:13:56Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
09:13:56.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-27T09:13:56Z Disabled control of system clock
09:13:56.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.23.244.254@KRBTEST.COM; defaulting to no policy
09:13:56.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:13:56.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.23.244.254@KRBTEST.COM" created.
09:13:56.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:13:56.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.23.244.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/kudu.keytab.
09:13:56.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.23.244.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/kudu.keytab.
09:13:56.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.23.244.254@KRBTEST.COM; defaulting to no policy
09:13:56.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:13:56.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.23.244.254@KRBTEST.COM" created.
09:13:56.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:13:56.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.23.244.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/kudu.keytab.
09:13:56.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.23.244.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/kudu.keytab.
09:13:56.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:56.260462 24531 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu
09:13:56.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu
09:13:56.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-0/wal
09:13:56.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-0/data
09:13:56.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:13:56.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:13:56.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:13:56.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:13:56.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:13:56.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:13:56.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-0/logs
09:13:56.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-0/data/info.pb
09:13:56.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:13:56.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:13:56.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:13:56.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:13:56.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:13:56.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:13:56.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:13:56.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:13:56.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:13:56.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:13:56.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:13:56.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:13:56.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.244.254:33613
09:13:56.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.244.254
09:13:56.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:13:56.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/kudu.keytab
09:13:56.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.244.254
09:13:56.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:13:56.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
09:13:56.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
09:13:56.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
09:13:56.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.244.212:44779
09:13:56.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:13:56.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:13:56.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:13:56.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:13:56.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.244.254:33613,127.23.244.253:40609,127.23.244.252:45583
09:13:56.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:13:56.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38985/jwks.json
09:13:56.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
09:13:56.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
09:13:56.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:56.517710 24560 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:13:56.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:56.518256 24560 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:13:56.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:56.518898 24560 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:13:56.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:56.546869 24560 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
09:13:56.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:56.547084 24560 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:13:56.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:56.547413 24560 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:13:56.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:56.547657 24560 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:13:56.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:56.547899 24560 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:13:56.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:56.548154 24560 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:13:56.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:56.548355 24560 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38985/jwks.json
09:13:56.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:56.580255 24560 master_runner.cc:386] Master server non-default flags:
09:13:56.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:13:56.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.244.212:44779
09:13:56.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:13:56.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:13:56.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-0/data
09:13:56.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-0/wal
09:13:56.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
09:13:56.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:13:56.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.244.254:33613,127.23.244.253:40609,127.23.244.252:45583
09:13:56.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/kudu.keytab
09:13:56.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.244.254
09:13:56.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:13:56.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:13:56.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:13:56.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.244.254:33613
09:13:56.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:13:56.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:13:56.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:13:56.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38985/jwks.json
09:13:56.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:13:56.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
09:13:56.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
09:13:56.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:13:56.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:13:56.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-0/data/info.pb
09:13:56.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.244.254
09:13:56.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:13:56.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:13:56.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:13:56.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:13:56.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:13:56.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:13:56.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-0/logs
09:13:56.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:13:56.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:13:56.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:13:56.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:13:56.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:13:56.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d0cb013a5aad754f2ed06c25f5ec3d8a1873622
09:13:56.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:13:56.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Oct 2025 08:43:21 UTC on 5fd53c4cbb9d
09:13:56.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8741
09:13:56.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:13:56.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:56.581547 24560 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:13:56.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:56.582626 24560 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:13:56.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:56.592279 24566 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:58.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:56.592918 24567 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:58.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:57.995596 24565 debug-util.cc:398] Leaking SignalData structure 0x7b0800027080 after lost signal to thread 24560
09:13:58.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:58.038617 24560 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.446s user 0.498s sys 0.897s
09:13:58.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:58.039758 24560 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.448s user 0.498s sys 0.897s
09:13:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:58.040912 24569 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:58.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:58.043494 24568 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1445 milliseconds
09:13:58.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:58.043583 24560 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:13:58.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:58.044644 24560 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:13:58.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:58.046693 24560 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:13:58.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:58.047988 24560 hybrid_clock.cc:648] HybridClock initialized: now 1761556438047949 us; error 36 us; skew 500 ppm
09:13:58.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:58.048686 24560 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:13:58.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 27 09:13:58 dist-test-slave-n41w krb5kdc[24542](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761556438, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.244.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:13:58.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:58.053478 24560 init.cc:377] Logged in from keytab as kudu/127.23.244.254@KRBTEST.COM (short username kudu)
09:13:58.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:58.057855 24560 webserver.cc:492] Webserver started at http://127.23.244.254:36337/ using document root <none> and password file <none>
09:13:58.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:58.058799 24560 fs_manager.cc:362] Metadata directory not provided
09:13:58.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:58.059048 24560 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:13:58.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:58.059471 24560 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
09:13:58.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:58.063175 24560 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-0/data/instance:
09:13:58.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6c69f67b25f8445fa1bed7d3e83853a0"
09:13:58.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-27 09:13:58 on dist-test-slave-n41w"
09:13:58.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:58.064148 24560 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-0/wal/instance:
09:13:58.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6c69f67b25f8445fa1bed7d3e83853a0"
09:13:58.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-27 09:13:58 on dist-test-slave-n41w"
09:13:58.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:58.070271 24560 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.005s sys 0.001s
09:13:58.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:58.074832 24578 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:13:58.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:58.075688 24560 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.001s sys 0.003s
09:13:58.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:58.075953 24560 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-0/data,/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-0/wal
09:13:58.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6c69f67b25f8445fa1bed7d3e83853a0"
09:13:58.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-27 09:13:58 on dist-test-slave-n41w"
09:13:58.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:58.076208 24560 fs_report.cc:389] FS layout report
09:13:58.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:13:58.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-0/wal
09:13:58.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-0/wal
09:13:58.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-0/data/data
09:13:58.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:13:58.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:13:58.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:13:58.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:13:58.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:13:58.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:13:58.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:13:58.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:13:58.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:13:58.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:13:58.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:13:58.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:58.115155 24560 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:13:58.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:58.124223 24560 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:13:58.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:58.124536 24560 kserver.cc:163] Server-wide thread pool size limit: 3276
09:13:58.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:58.185099 24560 rpc_server.cc:307] RPC server started. Bound to: 127.23.244.254:33613
09:13:58.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:58.185165 24629 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.244.254:33613 every 8 connection(s)
09:13:58.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:58.187403 24560 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-0/data/info.pb
09:13:58.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:58.192533 24531 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu as pid 24560
09:13:58.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:58.192608 24630 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
09:13:58.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:58.192874 24531 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-0/wal/instance
09:13:58.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:58.204496 24630 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.244.254" port: 33613 } has no permanent_uuid. Determining permanent_uuid...
09:13:58.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.23.244.253@KRBTEST.COM; defaulting to no policy
09:13:58.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:13:58.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.23.244.253@KRBTEST.COM" created.
09:13:58.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 27 09:13:58 dist-test-slave-n41w krb5kdc[24542](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761556438, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.244.254@KRBTEST.COM for kudu/127.23.244.254@KRBTEST.COM
09:13:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:13:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.23.244.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/kudu.keytab.
09:13:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.23.244.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/kudu.keytab.
09:13:58.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:58.241117 24630 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.244.253" port: 40609 } has no permanent_uuid. Determining permanent_uuid...
09:13:58.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:58.243454 24582 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.244.253:40609: connect: Connection refused (error 111)
09:13:58.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.23.244.253@KRBTEST.COM; defaulting to no policy
09:13:58.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:13:58.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.23.244.253@KRBTEST.COM" created.
09:13:58.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:58.247390 24630 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.244.253:40609: Network error: Client connection negotiation failed: client connection to 127.23.244.253:40609: connect: Connection refused (error 111)
09:13:58.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:13:58.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.23.244.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/kudu.keytab.
09:13:58.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.23.244.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/kudu.keytab.
09:13:58.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:58.265475 24531 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu
09:13:58.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu
09:13:58.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-1/wal
09:13:58.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-1/data
09:13:58.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:13:58.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:13:58.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:13:58.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:13:58.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:13:58.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:13:58.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-1/logs
09:13:58.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-1/data/info.pb
09:13:58.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:13:58.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:13:58.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:13:58.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:13:58.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:13:58.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:13:58.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:13:58.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:13:58.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:13:58.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:13:58.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:13:58.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:13:58.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.244.253:40609
09:13:58.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.244.253
09:13:58.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:13:58.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/kudu.keytab
09:13:58.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.244.253
09:13:58.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:13:58.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
09:13:58.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
09:13:58.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
09:13:58.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.244.212:44779
09:13:58.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:13:58.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:13:58.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:13:58.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:13:58.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.244.254:33613,127.23.244.253:40609,127.23.244.252:45583
09:13:58.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:13:58.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38985/jwks.json
09:13:58.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
09:13:58.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
09:13:58.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:58.296948 24630 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.244.253" port: 40609 } attempt: 1
09:13:58.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:58.300079 24630 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.244.253:40609: Network error: Client connection negotiation failed: client connection to 127.23.244.253:40609: connect: Connection refused (error 111)
09:13:58.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:58.368357 24630 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.244.253" port: 40609 } attempt: 2
09:13:58.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:58.371475 24630 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.244.253:40609: Network error: Client connection negotiation failed: client connection to 127.23.244.253:40609: connect: Connection refused (error 111)
09:13:58.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:58.462801 24630 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.244.253" port: 40609 } attempt: 3
09:13:58.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:58.465956 24630 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.244.253:40609: Network error: Client connection negotiation failed: client connection to 127.23.244.253:40609: connect: Connection refused (error 111)
09:13:58.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:58.551316 24639 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:13:58.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:58.551867 24639 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:13:58.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:58.552454 24639 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:13:58.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:58.579643 24639 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
09:13:58.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:58.579877 24639 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:13:58.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:58.580219 24639 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:13:58.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:58.580452 24639 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:13:58.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:58.580693 24639 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:13:58.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:58.580929 24639 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:13:58.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:58.581127 24639 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38985/jwks.json
09:13:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:58.609346 24630 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.244.253" port: 40609 } attempt: 4
09:13:58.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:58.612396 24630 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.244.253:40609: Network error: Client connection negotiation failed: client connection to 127.23.244.253:40609: connect: Connection refused (error 111)
09:13:58.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:58.612515 24639 master_runner.cc:386] Master server non-default flags:
09:13:58.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:13:58.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.244.212:44779
09:13:58.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:13:58.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:13:58.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-1/data
09:13:58.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-1/wal
09:13:58.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
09:13:58.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:13:58.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.244.254:33613,127.23.244.253:40609,127.23.244.252:45583
09:13:58.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/kudu.keytab
09:13:58.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.244.253
09:13:58.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:13:58.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:13:58.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:13:58.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.244.253:40609
09:13:58.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:13:58.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:13:58.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:13:58.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38985/jwks.json
09:13:58.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:13:58.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
09:13:58.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
09:13:58.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:13:58.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:13:58.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-1/data/info.pb
09:13:58.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.244.253
09:13:58.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:13:58.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:13:58.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:13:58.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:13:58.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:13:58.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:13:58.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-1/logs
09:13:58.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:13:58.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:13:58.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:13:58.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:13:58.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:13:58.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d0cb013a5aad754f2ed06c25f5ec3d8a1873622
09:13:58.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:13:58.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Oct 2025 08:43:21 UTC on 5fd53c4cbb9d
09:13:58.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8741
09:13:58.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:13:58.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:58.613524 24639 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:13:58.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:58.614327 24639 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:13:58.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:58.624136 24645 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:58.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:58.911800 24630 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.244.253" port: 40609 } attempt: 5
09:13:58.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:58.918784 24630 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.244.253:40609: Network error: Client connection negotiation failed: client connection to 127.23.244.253:40609: connect: Connection refused (error 111)
09:13:59.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:59.466291 24630 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.244.253" port: 40609 } attempt: 6
09:13:59.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:59.498785 24630 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.244.253:40609: Network error: Client connection negotiation failed: client connection to 127.23.244.253:40609: connect: Connection refused (error 111)
09:13:59.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:58.624461 24646 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:59.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:59.802651 24648 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:59.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:59.807164 24639 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.183s user 0.406s sys 0.749s
09:13:59.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:59.807492 24639 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.184s user 0.406s sys 0.749s
09:13:59.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:13:59.808302 24647 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1183 milliseconds
09:13:59.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:59.808316 24639 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:13:59.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:59.809571 24639 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:13:59.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:59.812292 24639 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:13:59.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:59.813678 24639 hybrid_clock.cc:648] HybridClock initialized: now 1761556439813590 us; error 75 us; skew 500 ppm
09:13:59.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:59.814388 24639 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:13:59.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 27 09:13:59 dist-test-slave-n41w krb5kdc[24542](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761556439, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.244.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:13:59.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:59.820334 24639 init.cc:377] Logged in from keytab as kudu/127.23.244.253@KRBTEST.COM (short username kudu)
09:13:59.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:59.826565 24639 webserver.cc:492] Webserver started at http://127.23.244.253:45357/ using document root <none> and password file <none>
09:13:59.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:59.827334 24639 fs_manager.cc:362] Metadata directory not provided
09:13:59.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:59.827526 24639 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:13:59.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:59.827956 24639 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
09:13:59.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:59.831717 24639 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-1/data/instance:
09:13:59.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4dc4196f0b844bcfbba87180c2307218"
09:13:59.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-27 09:13:59 on dist-test-slave-n41w"
09:13:59.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:59.832621 24639 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-1/wal/instance:
09:13:59.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4dc4196f0b844bcfbba87180c2307218"
09:13:59.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-27 09:13:59 on dist-test-slave-n41w"
09:13:59.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:59.840235 24639 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.003s sys 0.004s
09:13:59.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:59.845110 24658 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:13:59.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:59.846063 24639 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.005s sys 0.000s
09:13:59.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:59.846361 24639 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-1/data,/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-1/wal
09:13:59.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4dc4196f0b844bcfbba87180c2307218"
09:13:59.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-27 09:13:59 on dist-test-slave-n41w"
09:13:59.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:59.846632 24639 fs_report.cc:389] FS layout report
09:13:59.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:13:59.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-1/wal
09:13:59.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-1/wal
09:13:59.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-1/data/data
09:13:59.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:13:59.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:13:59.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:13:59.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:13:59.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:13:59.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:13:59.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:13:59.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:13:59.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:13:59.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:13:59.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:13:59.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:59.906288 24639 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:13:59.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:59.910948 24639 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:13:59.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:59.911368 24639 kserver.cc:163] Server-wide thread pool size limit: 3276
09:13:59.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:59.978976 24639 rpc_server.cc:307] RPC server started. Bound to: 127.23.244.253:40609
09:13:59.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:59.979044 24710 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.244.253:40609 every 8 connection(s)
09:13:59.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:59.981564 24639 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-1/data/info.pb
09:13:59.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:59.986649 24711 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
09:13:59.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:59.987918 24531 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu as pid 24639
09:13:59.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:59.988335 24531 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-1/wal/instance
09:14:00.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:13:59.998803 24711 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.244.254" port: 33613 } has no permanent_uuid. Determining permanent_uuid...
09:14:00.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.23.244.252@KRBTEST.COM; defaulting to no policy
09:14:00.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:14:00.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.23.244.252@KRBTEST.COM" created.
09:14:00.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 27 09:14:00 dist-test-slave-n41w krb5kdc[24542](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761556439, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.244.253@KRBTEST.COM for kudu/127.23.244.254@KRBTEST.COM
09:14:00.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:14:00.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.23.244.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/kudu.keytab.
09:14:00.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.23.244.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/kudu.keytab.
09:14:00.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:00.032635 24711 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.244.253" port: 40609 } has no permanent_uuid. Determining permanent_uuid...
09:14:00.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.23.244.252@KRBTEST.COM; defaulting to no policy
09:14:00.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:14:00.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.23.244.252@KRBTEST.COM" created.
09:14:00.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 27 09:14:00 dist-test-slave-n41w krb5kdc[24542](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761556439, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.244.253@KRBTEST.COM for kudu/127.23.244.253@KRBTEST.COM
09:14:00.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:00.053539 24711 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } has no permanent_uuid. Determining permanent_uuid...
09:14:00.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:00.055416 24661 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.244.252:45583: connect: Connection refused (error 111)
09:14:00.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:14:00.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.23.244.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/kudu.keytab.
09:14:00.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.23.244.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/kudu.keytab.
09:14:00.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:00.059168 24711 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.244.252:45583: Network error: Client connection negotiation failed: client connection to 127.23.244.252:45583: connect: Connection refused (error 111)
09:14:00.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:00.059501 24531 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu
09:14:00.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu
09:14:00.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-2/wal
09:14:00.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-2/data
09:14:00.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:14:00.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:14:00.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:14:00.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:14:00.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:14:00.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:14:00.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-2/logs
09:14:00.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-2/data/info.pb
09:14:00.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:14:00.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:14:00.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:14:00.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:14:00.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:14:00.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:14:00.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:14:00.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:14:00.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:14:00.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:14:00.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:14:00.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:14:00.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.244.252:45583
09:14:00.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.244.252
09:14:00.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:14:00.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/kudu.keytab
09:14:00.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.244.252
09:14:00.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:14:00.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
09:14:00.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
09:14:00.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
09:14:00.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.244.212:44779
09:14:00.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:14:00.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:14:00.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:14:00.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:14:00.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.244.254:33613,127.23.244.253:40609,127.23.244.252:45583
09:14:00.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:14:00.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38985/jwks.json
09:14:00.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
09:14:00.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
09:14:00.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:00.108590 24711 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } attempt: 1
09:14:00.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:00.111680 24711 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.244.252:45583: Network error: Client connection negotiation failed: client connection to 127.23.244.252:45583: connect: Connection refused (error 111)
09:14:00.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:00.180802 24711 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } attempt: 2
09:14:00.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:00.197291 24711 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.244.252:45583: Network error: Client connection negotiation failed: client connection to 127.23.244.252:45583: connect: Connection refused (error 111)
09:14:00.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:00.288677 24711 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } attempt: 3
09:14:00.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:00.291963 24711 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.244.252:45583: Network error: Client connection negotiation failed: client connection to 127.23.244.252:45583: connect: Connection refused (error 111)
09:14:00.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:00.342849 24721 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:14:00.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:00.343401 24721 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:14:00.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:00.344070 24721 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:14:00.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:00.370962 24721 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
09:14:00.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:00.371225 24721 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:14:00.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:00.371667 24721 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:14:00.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:00.371887 24721 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:14:00.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:00.372056 24721 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:14:00.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:00.372215 24721 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:14:00.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:00.372334 24721 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38985/jwks.json
09:14:00.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:00.403808 24721 master_runner.cc:386] Master server non-default flags:
09:14:00.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:14:00.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.244.212:44779
09:14:00.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:14:00.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:14:00.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-2/data
09:14:00.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-2/wal
09:14:00.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
09:14:00.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:14:00.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.244.254:33613,127.23.244.253:40609,127.23.244.252:45583
09:14:00.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/kudu.keytab
09:14:00.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.244.252
09:14:00.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:14:00.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:14:00.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:14:00.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.244.252:45583
09:14:00.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:14:00.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:14:00.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:14:00.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38985/jwks.json
09:14:00.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:14:00.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
09:14:00.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
09:14:00.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:14:00.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:14:00.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-2/data/info.pb
09:14:00.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.244.252
09:14:00.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:14:00.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:14:00.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:14:00.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:14:00.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:14:00.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:14:00.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-2/logs
09:14:00.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:14:00.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:14:00.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:14:00.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:14:00.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:14:00.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d0cb013a5aad754f2ed06c25f5ec3d8a1873622
09:14:00.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:14:00.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Oct 2025 08:43:21 UTC on 5fd53c4cbb9d
09:14:00.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8741
09:14:00.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:14:00.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:00.404815 24721 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:14:00.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:00.405678 24721 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:14:00.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:00.415159 24727 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:14:00.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:00.435249 24711 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } attempt: 4
09:14:00.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:00.441275 24711 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.244.252:45583: Network error: Client connection negotiation failed: client connection to 127.23.244.252:45583: connect: Connection refused (error 111)
09:14:00.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:00.559329 24630 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.244.253" port: 40609 } attempt: 7
09:14:00.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 27 09:14:00 dist-test-slave-n41w krb5kdc[24542](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761556438, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.244.254@KRBTEST.COM for kudu/127.23.244.253@KRBTEST.COM
09:14:00.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:00.616225 24630 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } has no permanent_uuid. Determining permanent_uuid...
09:14:00.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:00.639408 24630 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.244.252:45583: Network error: Client connection negotiation failed: client connection to 127.23.244.252:45583: connect: Connection refused (error 111)
09:14:00.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:00.697891 24630 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } attempt: 1
09:14:00.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:00.703557 24630 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.244.252:45583: Network error: Client connection negotiation failed: client connection to 127.23.244.252:45583: connect: Connection refused (error 111)
09:14:00.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:00.740729 24711 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } attempt: 5
09:14:00.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:00.746085 24711 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.244.252:45583: Network error: Client connection negotiation failed: client connection to 127.23.244.252:45583: connect: Connection refused (error 111)
09:14:00.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:00.785005 24630 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } attempt: 2
09:14:00.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:00.790556 24630 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.244.252:45583: Network error: Client connection negotiation failed: client connection to 127.23.244.252:45583: connect: Connection refused (error 111)
09:14:00.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:00.876001 24630 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } attempt: 3
09:14:00.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:00.882027 24630 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.244.252:45583: Network error: Client connection negotiation failed: client connection to 127.23.244.252:45583: connect: Connection refused (error 111)
09:14:01.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:01.022475 24630 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } attempt: 4
09:14:01.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:01.028656 24630 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.244.252:45583: Network error: Client connection negotiation failed: client connection to 127.23.244.252:45583: connect: Connection refused (error 111)
09:14:01.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:01.226563 24376 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad6a0 after lost signal to thread 24248
09:14:01.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:01.228013 24376 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad5c0 after lost signal to thread 24379
09:14:01.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:01.293511 24711 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } attempt: 6
09:14:01.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:01.312091 24630 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } attempt: 5
09:14:01.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:01.317713 24711 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.244.252:45583: Network error: Client connection negotiation failed: client connection to 127.23.244.252:45583: connect: Connection refused (error 111)
09:14:01.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:01.319758 24630 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.244.252:45583: Network error: Client connection negotiation failed: client connection to 127.23.244.252:45583: connect: Connection refused (error 111)
09:14:01.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:01.872224 24630 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } attempt: 6
09:14:02.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:02.026284 24580 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.GetNodeInstance -> {remote=127.23.244.252:45583, user_credentials={real_user=slave}} blocked reactor thread for 138654us
09:14:02.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:02.040354 24630 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.244.252:45583: Network error: Client connection negotiation failed: client connection to 127.23.244.252:45583: connect: Connection refused (error 111)
09:14:02.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:01.818192 24726 debug-util.cc:398] Leaking SignalData structure 0x7b0800027080 after lost signal to thread 24721
09:14:02.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:00.417069 24728 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:14:02.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:02.052656 24726 kernel_stack_watchdog.cc:198] Thread 24721 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
09:14:02.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
09:14:02.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
09:14:02.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
09:14:02.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
09:14:02.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:02.056221 24729 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1636 milliseconds
09:14:02.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:02.056197 24721 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.641s user 0.422s sys 0.916s
09:14:02.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:02.057169 24721 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.642s user 0.422s sys 0.916s
09:14:02.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:02.057909 24730 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:14:02.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.057897 24721 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:14:02.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.059218 24721 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:14:02.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.061810 24721 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:14:02.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.063174 24721 hybrid_clock.cc:648] HybridClock initialized: now 1761556442063103 us; error 69 us; skew 500 ppm
09:14:02.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.063921 24721 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:14:02.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 27 09:14:02 dist-test-slave-n41w krb5kdc[24542](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761556442, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.244.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:14:02.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.069305 24721 init.cc:377] Logged in from keytab as kudu/127.23.244.252@KRBTEST.COM (short username kudu)
09:14:02.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.074244 24721 webserver.cc:492] Webserver started at http://127.23.244.252:40373/ using document root <none> and password file <none>
09:14:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.075320 24721 fs_manager.cc:362] Metadata directory not provided
09:14:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.075578 24721 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:14:02.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.076064 24721 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
09:14:02.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.082026 24721 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-2/data/instance:
09:14:02.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "88fae1c8ea274939a7704e39fcf882f5"
09:14:02.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-27 09:14:02 on dist-test-slave-n41w"
09:14:02.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.083364 24721 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-2/wal/instance:
09:14:02.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "88fae1c8ea274939a7704e39fcf882f5"
09:14:02.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-27 09:14:02 on dist-test-slave-n41w"
09:14:02.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.092631 24721 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.005s sys 0.002s
09:14:02.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.099164 24749 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:14:02.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.100370 24721 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
09:14:02.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.100725 24721 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-2/data,/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-2/wal
09:14:02.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "88fae1c8ea274939a7704e39fcf882f5"
09:14:02.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-27 09:14:02 on dist-test-slave-n41w"
09:14:02.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.101155 24721 fs_report.cc:389] FS layout report
09:14:02.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:14:02.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-2/wal
09:14:02.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-2/wal
09:14:02.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-2/data/data
09:14:02.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:14:02.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:14:02.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:14:02.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:14:02.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:14:02.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:14:02.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:14:02.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:14:02.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:14:02.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:14:02.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:14:02.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.180675 24721 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:14:02.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.185845 24721 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:14:02.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.186240 24721 kserver.cc:163] Server-wide thread pool size limit: 3276
09:14:02.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.273160 24721 rpc_server.cc:307] RPC server started. Bound to: 127.23.244.252:45583
09:14:02.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.273236 24800 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.244.252:45583 every 8 connection(s)
09:14:02.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.275933 24721 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-2/data/info.pb
09:14:02.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.282001 24531 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu as pid 24721
09:14:02.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.282518 24531 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-2/wal/instance
09:14:02.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.300947 24801 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
09:14:02.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.316939 24801 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.244.254" port: 33613 } has no permanent_uuid. Determining permanent_uuid...
09:14:02.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 27 09:14:02 dist-test-slave-n41w krb5kdc[24542](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761556436, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.23.244.252@KRBTEST.COM
09:14:02.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 27 09:14:02 dist-test-slave-n41w krb5kdc[24542](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761556442, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.244.252@KRBTEST.COM for kudu/127.23.244.254@KRBTEST.COM
09:14:02.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.352097 24801 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.244.253" port: 40609 } has no permanent_uuid. Determining permanent_uuid...
09:14:02.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 27 09:14:02 dist-test-slave-n41w krb5kdc[24542](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761556436, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.23.244.253@KRBTEST.COM
09:14:02.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 27 09:14:02 dist-test-slave-n41w krb5kdc[24542](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761556442, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.244.252@KRBTEST.COM for kudu/127.23.244.253@KRBTEST.COM
09:14:02.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.374293 24801 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } has no permanent_uuid. Determining permanent_uuid...
09:14:02.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.378152 24711 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } attempt: 7
09:14:02.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 27 09:14:02 dist-test-slave-n41w krb5kdc[24542](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761556436, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.23.244.254@KRBTEST.COM
09:14:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 27 09:14:02 dist-test-slave-n41w krb5kdc[24542](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761556442, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.244.252@KRBTEST.COM for kudu/127.23.244.252@KRBTEST.COM
09:14:02.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 27 09:14:02 dist-test-slave-n41w krb5kdc[24542](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761556439, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.244.253@KRBTEST.COM for kudu/127.23.244.252@KRBTEST.COM
09:14:02.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.410394 24801 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5: Bootstrap starting.
09:14:02.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.411871 24531 external_mini_cluster.cc:949] 0 TS(s) registered with all masters
09:14:02.422 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:14:02.422 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
09:14:02.423 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
09:14:02.423 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
09:14:02.424 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.23.244.254"
port: 33613
}
bound_http_address {
host: "127.23.244.254"
port: 36337
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.23.244.253"
port: 40609
}
bound_http_address {
host: "127.23.244.253"
port: 45357
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.23.244.252"
port: 45583
}
bound_http_address {
host: "127.23.244.252"
port: 40373
}
}
}
09:14:02.425 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
09:14:02.425 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:14:02.425 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
09:14:02.425 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
09:14:02.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.425081 24801 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5: Neither blocks nor log segments found. Creating new log.
09:14:02.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.427361 24801 log.cc:826] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5: Log is configured to *not* fsync() on all Append() calls
09:14:02.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.428020 24711 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4dc4196f0b844bcfbba87180c2307218: Bootstrap starting.
09:14:02.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.443212 24711 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 4dc4196f0b844bcfbba87180c2307218: Neither blocks nor log segments found. Creating new log.
09:14:02.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.445399 24711 log.cc:826] T 00000000000000000000000000000000 P 4dc4196f0b844bcfbba87180c2307218: Log is configured to *not* fsync() on all Append() calls
09:14:02.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.451457 24801 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5: No bootstrap required, opened a new log
09:14:02.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 27 09:14:02 dist-test-slave-n41w krb5kdc[24542](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761556442, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:14:02.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
09:14:02.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.457998 24711 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4dc4196f0b844bcfbba87180c2307218: No bootstrap required, opened a new log
09:14:02.461 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:14:02.462 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/krb5cc
09:14:02.463 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
09:14:02.463 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/krb5cc
09:14:02.464 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
09:14:02.465 [DEBUG - Time-limited test] (SecurityContext.java:158) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
09:14:02.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.476528 24801 raft_consensus.cc:359] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c69f67b25f8445fa1bed7d3e83853a0" member_type: VOTER last_known_addr { host: "127.23.244.254" port: 33613 } } peers { permanent_uuid: "4dc4196f0b844bcfbba87180c2307218" member_type: VOTER last_known_addr { host: "127.23.244.253" port: 40609 } } peers { permanent_uuid: "88fae1c8ea274939a7704e39fcf882f5" member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } }
09:14:02.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.477258 24801 raft_consensus.cc:385] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:14:02.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.477527 24801 raft_consensus.cc:740] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 88fae1c8ea274939a7704e39fcf882f5, State: Initialized, Role: FOLLOWER
09:14:02.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.478451 24801 consensus_queue.cc:260] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c69f67b25f8445fa1bed7d3e83853a0" member_type: VOTER last_known_addr { host: "127.23.244.254" port: 33613 } } peers { permanent_uuid: "4dc4196f0b844bcfbba87180c2307218" member_type: VOTER last_known_addr { host: "127.23.244.253" port: 40609 } } peers { permanent_uuid: "88fae1c8ea274939a7704e39fcf882f5" member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } }
09:14:02.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.482223 24815 sys_catalog.cc:455] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c69f67b25f8445fa1bed7d3e83853a0" member_type: VOTER last_known_addr { host: "127.23.244.254" port: 33613 } } peers { permanent_uuid: "4dc4196f0b844bcfbba87180c2307218" member_type: VOTER last_known_addr { host: "127.23.244.253" port: 40609 } } peers { permanent_uuid: "88fae1c8ea274939a7704e39fcf882f5" member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } } }
09:14:02.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.483008 24815 sys_catalog.cc:458] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5 [sys.catalog]: This master's current role is: FOLLOWER
09:14:02.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.483959 24801 sys_catalog.cc:565] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5 [sys.catalog]: configured and running, proceeding with master startup.
09:14:02.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.485531 24711 raft_consensus.cc:359] T 00000000000000000000000000000000 P 4dc4196f0b844bcfbba87180c2307218 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c69f67b25f8445fa1bed7d3e83853a0" member_type: VOTER last_known_addr { host: "127.23.244.254" port: 33613 } } peers { permanent_uuid: "4dc4196f0b844bcfbba87180c2307218" member_type: VOTER last_known_addr { host: "127.23.244.253" port: 40609 } } peers { permanent_uuid: "88fae1c8ea274939a7704e39fcf882f5" member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } }
09:14:02.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.486050 24711 raft_consensus.cc:385] T 00000000000000000000000000000000 P 4dc4196f0b844bcfbba87180c2307218 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:14:02.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.486392 24711 raft_consensus.cc:740] T 00000000000000000000000000000000 P 4dc4196f0b844bcfbba87180c2307218 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4dc4196f0b844bcfbba87180c2307218, State: Initialized, Role: FOLLOWER
09:14:02.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.487149 24711 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4dc4196f0b844bcfbba87180c2307218 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c69f67b25f8445fa1bed7d3e83853a0" member_type: VOTER last_known_addr { host: "127.23.244.254" port: 33613 } } peers { permanent_uuid: "4dc4196f0b844bcfbba87180c2307218" member_type: VOTER last_known_addr { host: "127.23.244.253" port: 40609 } } peers { permanent_uuid: "88fae1c8ea274939a7704e39fcf882f5" member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } }
09:14:02.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.495707 24818 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4dc4196f0b844bcfbba87180c2307218 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c69f67b25f8445fa1bed7d3e83853a0" member_type: VOTER last_known_addr { host: "127.23.244.254" port: 33613 } } peers { permanent_uuid: "4dc4196f0b844bcfbba87180c2307218" member_type: VOTER last_known_addr { host: "127.23.244.253" port: 40609 } } peers { permanent_uuid: "88fae1c8ea274939a7704e39fcf882f5" member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } } }
09:14:02.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.496464 24818 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4dc4196f0b844bcfbba87180c2307218 [sys.catalog]: This master's current role is: FOLLOWER
09:14:02.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.497491 24711 sys_catalog.cc:565] T 00000000000000000000000000000000 P 4dc4196f0b844bcfbba87180c2307218 [sys.catalog]: configured and running, proceeding with master startup.
09:14:02.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.550561 24818 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4dc4196f0b844bcfbba87180c2307218 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:14:02.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.551302 24818 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4dc4196f0b844bcfbba87180c2307218 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c69f67b25f8445fa1bed7d3e83853a0" member_type: VOTER last_known_addr { host: "127.23.244.254" port: 33613 } } peers { permanent_uuid: "4dc4196f0b844bcfbba87180c2307218" member_type: VOTER last_known_addr { host: "127.23.244.253" port: 40609 } } peers { permanent_uuid: "88fae1c8ea274939a7704e39fcf882f5" member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } }
09:14:02.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.555497 24831 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
09:14:02.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.558317 24776 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4dc4196f0b844bcfbba87180c2307218" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "88fae1c8ea274939a7704e39fcf882f5" is_pre_election: true
09:14:02.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.559146 24776 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4dc4196f0b844bcfbba87180c2307218 in term 0.
09:14:02.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.560506 24661 leader_election.cc:304] T 00000000000000000000000000000000 P 4dc4196f0b844bcfbba87180c2307218 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4dc4196f0b844bcfbba87180c2307218, 88fae1c8ea274939a7704e39fcf882f5; no voters:
09:14:02.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.557983 24818 leader_election.cc:290] T 00000000000000000000000000000000 P 4dc4196f0b844bcfbba87180c2307218 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6c69f67b25f8445fa1bed7d3e83853a0 (127.23.244.254:33613), 88fae1c8ea274939a7704e39fcf882f5 (127.23.244.252:45583)
09:14:02.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.564082 24818 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 4dc4196f0b844bcfbba87180c2307218 [term 0 FOLLOWER]: Leader pre-election won for term 1
09:14:02.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.564590 24818 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4dc4196f0b844bcfbba87180c2307218 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
09:14:02.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.565044 24818 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 4dc4196f0b844bcfbba87180c2307218 [term 0 FOLLOWER]: Advancing to term 1
09:14:02.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.582816 24818 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4dc4196f0b844bcfbba87180c2307218 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c69f67b25f8445fa1bed7d3e83853a0" member_type: VOTER last_known_addr { host: "127.23.244.254" port: 33613 } } peers { permanent_uuid: "4dc4196f0b844bcfbba87180c2307218" member_type: VOTER last_known_addr { host: "127.23.244.253" port: 40609 } } peers { permanent_uuid: "88fae1c8ea274939a7704e39fcf882f5" member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } }
09:14:02.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.590637 24776 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4dc4196f0b844bcfbba87180c2307218" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "88fae1c8ea274939a7704e39fcf882f5"
09:14:02.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.591113 24776 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5 [term 0 FOLLOWER]: Advancing to term 1
09:14:02.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.598255 24776 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4dc4196f0b844bcfbba87180c2307218 in term 1.
09:14:02.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.599208 24661 leader_election.cc:304] T 00000000000000000000000000000000 P 4dc4196f0b844bcfbba87180c2307218 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4dc4196f0b844bcfbba87180c2307218, 88fae1c8ea274939a7704e39fcf882f5; no voters:
09:14:02.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.600050 24832 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 4dc4196f0b844bcfbba87180c2307218 [term 1 FOLLOWER]: Leader election won for term 1
09:14:02.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.600140 24818 leader_election.cc:290] T 00000000000000000000000000000000 P 4dc4196f0b844bcfbba87180c2307218 [CANDIDATE]: Term 1 election: Requested vote from peers 6c69f67b25f8445fa1bed7d3e83853a0 (127.23.244.254:33613), 88fae1c8ea274939a7704e39fcf882f5 (127.23.244.252:45583)
09:14:02.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.605141 24832 raft_consensus.cc:697] T 00000000000000000000000000000000 P 4dc4196f0b844bcfbba87180c2307218 [term 1 LEADER]: Becoming Leader. State: Replica: 4dc4196f0b844bcfbba87180c2307218, State: Running, Role: LEADER
09:14:02.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.562901 24721 master_runner.cc:186] Error getting master registration for 127.23.244.254:33613: OK, instance_id { permanent_uuid: "6c69f67b25f8445fa1bed7d3e83853a0" instance_seqno: 1761556438168136 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
09:14:02.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.606735 24721 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
09:14:02.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.607229 24832 consensus_queue.cc:237] T 00000000000000000000000000000000 P 4dc4196f0b844bcfbba87180c2307218 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c69f67b25f8445fa1bed7d3e83853a0" member_type: VOTER last_known_addr { host: "127.23.244.254" port: 33613 } } peers { permanent_uuid: "4dc4196f0b844bcfbba87180c2307218" member_type: VOTER last_known_addr { host: "127.23.244.253" port: 40609 } } peers { permanent_uuid: "88fae1c8ea274939a7704e39fcf882f5" member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } }
09:14:02.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.616725 24818 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4dc4196f0b844bcfbba87180c2307218 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4dc4196f0b844bcfbba87180c2307218. Latest consensus state: current_term: 1 leader_uuid: "4dc4196f0b844bcfbba87180c2307218" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c69f67b25f8445fa1bed7d3e83853a0" member_type: VOTER last_known_addr { host: "127.23.244.254" port: 33613 } } peers { permanent_uuid: "4dc4196f0b844bcfbba87180c2307218" member_type: VOTER last_known_addr { host: "127.23.244.253" port: 40609 } } peers { permanent_uuid: "88fae1c8ea274939a7704e39fcf882f5" member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } } }
09:14:02.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.618721 24818 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4dc4196f0b844bcfbba87180c2307218 [sys.catalog]: This master's current role is: LEADER
09:14:02.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.622335 24838 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
09:14:02.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.638947 24838 catalog_manager.cc:1494] Initializing Kudu cluster ID...
09:14:02.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.660924 24776 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5 [term 1 FOLLOWER]: Refusing update from remote peer 4dc4196f0b844bcfbba87180c2307218: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
09:14:02.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.663532 24832 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 4dc4196f0b844bcfbba87180c2307218 [LEADER]: Connected to new peer: Peer: permanent_uuid: "88fae1c8ea274939a7704e39fcf882f5" member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
09:14:02.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:02.670503 24665 tablet.cc:2378] T 00000000000000000000000000000000 P 4dc4196f0b844bcfbba87180c2307218: Can't schedule compaction. Clean time has not been advanced past its initial value.
09:14:02.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:02.691906 24755 tablet.cc:2378] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5: Can't schedule compaction. Clean time has not been advanced past its initial value.
09:14:02.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.700681 24815 sys_catalog.cc:455] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4dc4196f0b844bcfbba87180c2307218. Latest consensus state: current_term: 1 leader_uuid: "4dc4196f0b844bcfbba87180c2307218" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c69f67b25f8445fa1bed7d3e83853a0" member_type: VOTER last_known_addr { host: "127.23.244.254" port: 33613 } } peers { permanent_uuid: "4dc4196f0b844bcfbba87180c2307218" member_type: VOTER last_known_addr { host: "127.23.244.253" port: 40609 } } peers { permanent_uuid: "88fae1c8ea274939a7704e39fcf882f5" member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } } }
09:14:02.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.703142 24815 sys_catalog.cc:458] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5 [sys.catalog]: This master's current role is: FOLLOWER
09:14:02.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.707885 24844 mvcc.cc:204] Tried to move back new op lower bound from 7215335189117501440 to 7215335188929937408. Current Snapshot: MvccSnapshot[applied={T|T < 7215335189117501440}]
09:14:02.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.713091 24818 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4dc4196f0b844bcfbba87180c2307218 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "4dc4196f0b844bcfbba87180c2307218" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c69f67b25f8445fa1bed7d3e83853a0" member_type: VOTER last_known_addr { host: "127.23.244.254" port: 33613 } } peers { permanent_uuid: "4dc4196f0b844bcfbba87180c2307218" member_type: VOTER last_known_addr { host: "127.23.244.253" port: 40609 } } peers { permanent_uuid: "88fae1c8ea274939a7704e39fcf882f5" member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } } }
09:14:02.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.713681 24818 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4dc4196f0b844bcfbba87180c2307218 [sys.catalog]: This master's current role is: LEADER
09:14:02.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.715200 24838 catalog_manager.cc:1357] Generated new cluster ID: 34126f028d4542fdb530af64d91fa79a
09:14:02.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.715427 24838 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
09:14:02.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.725251 24815 sys_catalog.cc:455] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "4dc4196f0b844bcfbba87180c2307218" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c69f67b25f8445fa1bed7d3e83853a0" member_type: VOTER last_known_addr { host: "127.23.244.254" port: 33613 } } peers { permanent_uuid: "4dc4196f0b844bcfbba87180c2307218" member_type: VOTER last_known_addr { host: "127.23.244.253" port: 40609 } } peers { permanent_uuid: "88fae1c8ea274939a7704e39fcf882f5" member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } } }
09:14:02.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.725788 24815 sys_catalog.cc:458] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5 [sys.catalog]: This master's current role is: FOLLOWER
09:14:02.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.655468 24639 master_runner.cc:186] Error getting master registration for 127.23.244.254:33613: OK, instance_id { permanent_uuid: "6c69f67b25f8445fa1bed7d3e83853a0" instance_seqno: 1761556438168136 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
09:14:02.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.744614 24639 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
09:14:02.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.748375 24838 catalog_manager.cc:1380] Generated new certificate authority record
09:14:02.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.750092 24838 catalog_manager.cc:1514] Loading token signing keys...
09:14:02.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.774077 24838 catalog_manager.cc:6022] T 00000000000000000000000000000000 P 4dc4196f0b844bcfbba87180c2307218: Generated new TSK 0
09:14:02.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:02.774879 24838 catalog_manager.cc:1524] Initializing in-progress tserver states...
09:14:02.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:02.995841 24233 debug-util.cc:398] Leaking SignalData structure 0x7b08000b2d60 after lost signal to thread 24100
09:14:02.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:02.996798 24233 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad200 after lost signal to thread 24236
09:14:03.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:03.073813 24630 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } attempt: 7
09:14:03.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 27 09:14:03 dist-test-slave-n41w krb5kdc[24542](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761556438, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.244.254@KRBTEST.COM for kudu/127.23.244.252@KRBTEST.COM
09:14:03.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:03.126525 24630 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0: Bootstrap starting.
09:14:03.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:03.140293 24630 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0: Neither blocks nor log segments found. Creating new log.
09:14:03.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:03.144392 24630 log.cc:826] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0: Log is configured to *not* fsync() on all Append() calls
09:14:03.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:03.157970 24630 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0: No bootstrap required, opened a new log
09:14:03.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:03.213546 24630 raft_consensus.cc:359] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c69f67b25f8445fa1bed7d3e83853a0" member_type: VOTER last_known_addr { host: "127.23.244.254" port: 33613 } } peers { permanent_uuid: "4dc4196f0b844bcfbba87180c2307218" member_type: VOTER last_known_addr { host: "127.23.244.253" port: 40609 } } peers { permanent_uuid: "88fae1c8ea274939a7704e39fcf882f5" member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } }
09:14:03.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:03.215456 24630 raft_consensus.cc:385] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:14:03.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:03.216285 24630 raft_consensus.cc:740] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6c69f67b25f8445fa1bed7d3e83853a0, State: Initialized, Role: FOLLOWER
09:14:03.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:03.218930 24630 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c69f67b25f8445fa1bed7d3e83853a0" member_type: VOTER last_known_addr { host: "127.23.244.254" port: 33613 } } peers { permanent_uuid: "4dc4196f0b844bcfbba87180c2307218" member_type: VOTER last_known_addr { host: "127.23.244.253" port: 40609 } } peers { permanent_uuid: "88fae1c8ea274939a7704e39fcf882f5" member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } }
09:14:03.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:03.238963 24853 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c69f67b25f8445fa1bed7d3e83853a0" member_type: VOTER last_known_addr { host: "127.23.244.254" port: 33613 } } peers { permanent_uuid: "4dc4196f0b844bcfbba87180c2307218" member_type: VOTER last_known_addr { host: "127.23.244.253" port: 40609 } } peers { permanent_uuid: "88fae1c8ea274939a7704e39fcf882f5" member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } } }
09:14:03.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:03.239697 24853 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0 [sys.catalog]: This master's current role is: FOLLOWER
09:14:03.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:03.245420 24603 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0 [term 0 FOLLOWER]: Advancing to term 1
09:14:03.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:03.248456 24604 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4dc4196f0b844bcfbba87180c2307218" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6c69f67b25f8445fa1bed7d3e83853a0"
09:14:03.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:03.249681 24605 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4dc4196f0b844bcfbba87180c2307218" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6c69f67b25f8445fa1bed7d3e83853a0" is_pre_election: true
09:14:03.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:03.255502 24603 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0 [term 1 FOLLOWER]: Refusing update from remote peer 4dc4196f0b844bcfbba87180c2307218: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
09:14:03.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:03.256909 24818 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 4dc4196f0b844bcfbba87180c2307218 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6c69f67b25f8445fa1bed7d3e83853a0" member_type: VOTER last_known_addr { host: "127.23.244.254" port: 33613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:14:03.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:03.276794 24630 sys_catalog.cc:565] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0 [sys.catalog]: configured and running, proceeding with master startup.
09:14:03.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:03.343951 24853 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4dc4196f0b844bcfbba87180c2307218. Latest consensus state: current_term: 1 leader_uuid: "4dc4196f0b844bcfbba87180c2307218" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c69f67b25f8445fa1bed7d3e83853a0" member_type: VOTER last_known_addr { host: "127.23.244.254" port: 33613 } } peers { permanent_uuid: "4dc4196f0b844bcfbba87180c2307218" member_type: VOTER last_known_addr { host: "127.23.244.253" port: 40609 } } peers { permanent_uuid: "88fae1c8ea274939a7704e39fcf882f5" member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } } }
09:14:03.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:03.344607 24853 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0 [sys.catalog]: This master's current role is: FOLLOWER
09:14:03.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:03.344045 24859 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "4dc4196f0b844bcfbba87180c2307218" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c69f67b25f8445fa1bed7d3e83853a0" member_type: VOTER last_known_addr { host: "127.23.244.254" port: 33613 } } peers { permanent_uuid: "4dc4196f0b844bcfbba87180c2307218" member_type: VOTER last_known_addr { host: "127.23.244.253" port: 40609 } } peers { permanent_uuid: "88fae1c8ea274939a7704e39fcf882f5" member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } } }
09:14:03.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:03.345043 24859 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0 [sys.catalog]: This master's current role is: FOLLOWER
09:14:03.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:03.396027 24869 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
09:14:03.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:03.562816 24831 catalog_manager.cc:1269] Loaded cluster ID: 34126f028d4542fdb530af64d91fa79a
09:14:03.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:03.563220 24831 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5: loading cluster ID for follower catalog manager: success
09:14:03.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:03.569301 24831 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5: acquiring CA information for follower catalog manager: success
09:14:03.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:03.574136 24831 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:14:03.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:03.727483 24871 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4dc4196f0b844bcfbba87180c2307218 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "4dc4196f0b844bcfbba87180c2307218" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c69f67b25f8445fa1bed7d3e83853a0" member_type: VOTER last_known_addr { host: "127.23.244.254" port: 33613 } } peers { permanent_uuid: "4dc4196f0b844bcfbba87180c2307218" member_type: VOTER last_known_addr { host: "127.23.244.253" port: 40609 } } peers { permanent_uuid: "88fae1c8ea274939a7704e39fcf882f5" member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } } }
09:14:03.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:03.728806 24871 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4dc4196f0b844bcfbba87180c2307218 [sys.catalog]: This master's current role is: LEADER
09:14:04.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:04.415685 24869 catalog_manager.cc:1269] Loaded cluster ID: 34126f028d4542fdb530af64d91fa79a
09:14:04.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:04.416090 24869 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0: loading cluster ID for follower catalog manager: success
09:14:04.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:04.422744 24869 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0: acquiring CA information for follower catalog manager: success
09:14:04.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:04.428858 24869 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:14:05.465 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.23.244.254:33613
09:14:05.465 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
09:14:05.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:05.466444 24531 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu with pid 24560
09:14:05.490 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:14:05.490 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.23.244.252:45583
09:14:05.490 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
09:14:05.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:05.490887 24531 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu with pid 24721
09:14:05.517 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:14:05.517 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.23.244.253:40609
09:14:05.518 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
09:14:05.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:05.518251 24531 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu with pid 24639
09:14:05.548 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:14:05.549 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.23.244.254:33613
09:14:05.549 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
09:14:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:05.550338 24531 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu
09:14:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu
09:14:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-0/wal
09:14:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-0/data
09:14:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:14:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:14:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:14:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:14:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:14:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:14:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-0/logs
09:14:05.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-0/data/info.pb
09:14:05.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:14:05.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:14:05.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:14:05.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:14:05.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:14:05.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:14:05.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:14:05.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:14:05.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:14:05.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:14:05.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:14:05.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:14:05.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.244.254:33613
09:14:05.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.244.254
09:14:05.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36337
09:14:05.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/kudu.keytab
09:14:05.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.244.254
09:14:05.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:14:05.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
09:14:05.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
09:14:05.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
09:14:05.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.244.212:44779
09:14:05.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:14:05.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:14:05.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:14:05.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:14:05.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.244.254:33613,127.23.244.253:40609,127.23.244.252:45583
09:14:05.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:14:05.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38985/jwks.json
09:14:05.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
09:14:05.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
09:14:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:05.810844 24877 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:14:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:05.811267 24877 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:14:05.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:05.811921 24877 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:14:05.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:05.838634 24877 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
09:14:05.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:05.838800 24877 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:14:05.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:05.839025 24877 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:14:05.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:05.839188 24877 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:14:05.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:05.839347 24877 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:14:05.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:05.839536 24877 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:14:05.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:05.839761 24877 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38985/jwks.json
09:14:05.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:05.871470 24877 master_runner.cc:386] Master server non-default flags:
09:14:05.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:14:05.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.244.212:44779
09:14:05.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:14:05.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:14:05.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-0/data
09:14:05.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-0/wal
09:14:05.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
09:14:05.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:14:05.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.244.254:33613,127.23.244.253:40609,127.23.244.252:45583
09:14:05.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/kudu.keytab
09:14:05.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.244.254
09:14:05.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:14:05.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:14:05.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:14:05.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.244.254:33613
09:14:05.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:14:05.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:14:05.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:14:05.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38985/jwks.json
09:14:05.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:14:05.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
09:14:05.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
09:14:05.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:14:05.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:14:05.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-0/data/info.pb
09:14:05.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.244.254
09:14:05.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36337
09:14:05.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:14:05.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:14:05.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:14:05.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:14:05.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:14:05.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-0/logs
09:14:05.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:14:05.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:14:05.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:14:05.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:14:05.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:14:05.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d0cb013a5aad754f2ed06c25f5ec3d8a1873622
09:14:05.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:14:05.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Oct 2025 08:43:21 UTC on 5fd53c4cbb9d
09:14:05.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8741
09:14:05.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:14:05.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:05.872748 24877 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:14:05.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:05.873651 24877 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:14:05.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:05.883919 24883 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:14:07.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:05.884359 24884 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:14:07.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:07.177505 24886 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:14:07.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:07.180227 24885 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1295 milliseconds
09:14:07.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:07.181022 24877 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.297s user 0.426s sys 0.853s
09:14:07.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:07.181233 24877 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.297s user 0.426s sys 0.853s
09:14:07.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:07.181380 24877 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:14:07.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:07.182360 24877 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:14:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:07.184741 24877 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:14:07.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:07.186072 24877 hybrid_clock.cc:648] HybridClock initialized: now 1761556447186038 us; error 32 us; skew 500 ppm
09:14:07.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:07.186861 24877 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:14:07.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 27 09:14:07 dist-test-slave-n41w krb5kdc[24542](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761556447, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.244.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:14:07.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:07.193051 24877 init.cc:377] Logged in from keytab as kudu/127.23.244.254@KRBTEST.COM (short username kudu)
09:14:07.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:07.197846 24877 webserver.cc:492] Webserver started at http://127.23.244.254:36337/ using document root <none> and password file <none>
09:14:07.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:07.198643 24877 fs_manager.cc:362] Metadata directory not provided
09:14:07.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:07.198845 24877 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:14:07.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:07.206548 24877 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.006s sys 0.000s
09:14:07.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:07.210748 24894 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:14:07.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:07.211826 24877 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
09:14:07.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:07.212107 24877 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-0/data,/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-0/wal
09:14:07.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6c69f67b25f8445fa1bed7d3e83853a0"
09:14:07.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-27 09:13:58 on dist-test-slave-n41w"
09:14:07.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:07.213693 24877 fs_report.cc:389] FS layout report
09:14:07.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:14:07.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-0/wal
09:14:07.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-0/wal
09:14:07.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-0/data/data
09:14:07.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:14:07.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:14:07.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:14:07.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:14:07.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:14:07.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:14:07.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:14:07.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:14:07.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:14:07.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:14:07.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:14:07.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:07.271456 24877 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:14:07.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:07.277444 24877 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:14:07.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:07.277882 24877 kserver.cc:163] Server-wide thread pool size limit: 3276
09:14:07.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:07.341295 24877 rpc_server.cc:307] RPC server started. Bound to: 127.23.244.254:33613
09:14:07.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:07.341364 24945 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.244.254:33613 every 8 connection(s)
09:14:07.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:07.343802 24877 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-0/data/info.pb
09:14:07.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:07.352759 24946 sys_catalog.cc:263] Verifying existing consensus state
09:14:07.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:07.352864 24531 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu as pid 24877
09:14:07.353 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:14:07.353 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.23.244.252:45583
09:14:07.353 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
09:14:07.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:07.354496 24531 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu
09:14:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu
09:14:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-2/wal
09:14:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-2/data
09:14:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:14:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:14:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:14:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:14:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:14:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:14:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-2/logs
09:14:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-2/data/info.pb
09:14:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:14:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:14:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:14:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:14:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:14:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:14:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:14:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:14:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:14:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:14:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:14:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:14:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.244.252:45583
09:14:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.244.252
09:14:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40373
09:14:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/kudu.keytab
09:14:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.244.252
09:14:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:14:07.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
09:14:07.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
09:14:07.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
09:14:07.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.244.212:44779
09:14:07.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:14:07.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:14:07.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:14:07.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:14:07.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.244.254:33613,127.23.244.253:40609,127.23.244.252:45583
09:14:07.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:14:07.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38985/jwks.json
09:14:07.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
09:14:07.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
09:14:07.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:07.357985 24946 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0: Bootstrap starting.
09:14:07.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:07.403167 24946 log.cc:826] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0: Log is configured to *not* fsync() on all Append() calls
09:14:07.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:07.418998 24946 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
09:14:07.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:07.419915 24946 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0: Bootstrap complete.
09:14:07.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:07.448184 24946 raft_consensus.cc:359] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c69f67b25f8445fa1bed7d3e83853a0" member_type: VOTER last_known_addr { host: "127.23.244.254" port: 33613 } } peers { permanent_uuid: "4dc4196f0b844bcfbba87180c2307218" member_type: VOTER last_known_addr { host: "127.23.244.253" port: 40609 } } peers { permanent_uuid: "88fae1c8ea274939a7704e39fcf882f5" member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } }
09:14:07.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:07.451309 24946 raft_consensus.cc:740] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6c69f67b25f8445fa1bed7d3e83853a0, State: Initialized, Role: FOLLOWER
09:14:07.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:07.452154 24946 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c69f67b25f8445fa1bed7d3e83853a0" member_type: VOTER last_known_addr { host: "127.23.244.254" port: 33613 } } peers { permanent_uuid: "4dc4196f0b844bcfbba87180c2307218" member_type: VOTER last_known_addr { host: "127.23.244.253" port: 40609 } } peers { permanent_uuid: "88fae1c8ea274939a7704e39fcf882f5" member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } }
09:14:07.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:07.454484 24950 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c69f67b25f8445fa1bed7d3e83853a0" member_type: VOTER last_known_addr { host: "127.23.244.254" port: 33613 } } peers { permanent_uuid: "4dc4196f0b844bcfbba87180c2307218" member_type: VOTER last_known_addr { host: "127.23.244.253" port: 40609 } } peers { permanent_uuid: "88fae1c8ea274939a7704e39fcf882f5" member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } } }
09:14:07.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:07.454984 24950 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0 [sys.catalog]: This master's current role is: FOLLOWER
09:14:07.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:07.456483 24946 sys_catalog.cc:565] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0 [sys.catalog]: configured and running, proceeding with master startup.
09:14:07.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:07.489908 24961 catalog_manager.cc:1269] Loaded cluster ID: 34126f028d4542fdb530af64d91fa79a
09:14:07.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:07.490140 24961 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0: loading cluster ID for follower catalog manager: success
09:14:07.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:07.496766 24961 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0: acquiring CA information for follower catalog manager: success
09:14:07.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 27 09:14:07 dist-test-slave-n41w krb5kdc[24542](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761556447, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.244.254@KRBTEST.COM for kudu/127.23.244.254@KRBTEST.COM
09:14:07.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:07.502398 24961 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:14:07.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:07.543056 24233 debug-util.cc:398] Leaking SignalData structure 0x7b08000b8920 after lost signal to thread 24100
09:14:07.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:07.544267 24233 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad9c0 after lost signal to thread 24236
09:14:07.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:07.546051 24877 master_runner.cc:186] Error getting master registration for 127.23.244.253:40609: Network error: Client connection negotiation failed: client connection to 127.23.244.253:40609: connect: Connection refused (error 111),
09:14:07.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:07.571228 24877 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
09:14:07.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:07.688162 24948 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:14:07.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:07.688697 24948 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:14:07.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:07.689440 24948 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:14:07.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:07.720675 24948 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
09:14:07.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:07.720860 24948 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:14:07.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:07.721168 24948 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:14:07.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:07.721342 24948 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:14:07.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:07.721503 24948 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:14:07.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:07.721668 24948 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:14:07.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:07.721789 24948 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38985/jwks.json
09:14:07.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:07.756539 24948 master_runner.cc:386] Master server non-default flags:
09:14:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:14:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.244.212:44779
09:14:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:14:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:14:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-2/data
09:14:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-2/wal
09:14:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
09:14:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:14:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.244.254:33613,127.23.244.253:40609,127.23.244.252:45583
09:14:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/kudu.keytab
09:14:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.244.252
09:14:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:14:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:14:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:14:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.244.252:45583
09:14:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:14:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:14:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:14:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38985/jwks.json
09:14:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:14:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
09:14:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
09:14:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:14:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:14:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-2/data/info.pb
09:14:07.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.244.252
09:14:07.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40373
09:14:07.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:14:07.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:14:07.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:14:07.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:14:07.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:14:07.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-2/logs
09:14:07.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:14:07.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:14:07.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:14:07.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:14:07.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:14:07.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d0cb013a5aad754f2ed06c25f5ec3d8a1873622
09:14:07.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:14:07.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Oct 2025 08:43:21 UTC on 5fd53c4cbb9d
09:14:07.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8741
09:14:07.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:14:07.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:07.757751 24948 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:14:07.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:07.758975 24948 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:14:07.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:07.770534 24972 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:14:09.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.162071 24980 raft_consensus.cc:493] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:14:09.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.162930 24980 raft_consensus.cc:515] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c69f67b25f8445fa1bed7d3e83853a0" member_type: VOTER last_known_addr { host: "127.23.244.254" port: 33613 } } peers { permanent_uuid: "4dc4196f0b844bcfbba87180c2307218" member_type: VOTER last_known_addr { host: "127.23.244.253" port: 40609 } } peers { permanent_uuid: "88fae1c8ea274939a7704e39fcf882f5" member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } }
09:14:09.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.179400 24980 leader_election.cc:290] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 4dc4196f0b844bcfbba87180c2307218 (127.23.244.253:40609), 88fae1c8ea274939a7704e39fcf882f5 (127.23.244.252:45583)
09:14:09.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:09.187422 24896 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.244.252:45583: connect: Connection refused (error 111)
09:14:09.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:09.200543 24898 leader_election.cc:336] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 4dc4196f0b844bcfbba87180c2307218 (127.23.244.253:40609): Network error: Client connection negotiation failed: client connection to 127.23.244.253:40609: connect: Connection refused (error 111)
09:14:09.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:09.202940 24896 leader_election.cc:336] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 88fae1c8ea274939a7704e39fcf882f5 (127.23.244.252:45583): Network error: Client connection negotiation failed: client connection to 127.23.244.252:45583: connect: Connection refused (error 111)
09:14:09.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.203369 24896 leader_election.cc:304] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6c69f67b25f8445fa1bed7d3e83853a0; no voters: 4dc4196f0b844bcfbba87180c2307218, 88fae1c8ea274939a7704e39fcf882f5
09:14:09.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.204140 24980 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
09:14:09.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:07.775192 24975 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:14:09.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:07.772161 24973 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:14:09.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:09.214998 24974 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1440 milliseconds
09:14:09.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.215113 24948 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:14:09.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.216362 24948 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:14:09.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.219381 24948 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:14:09.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.220814 24948 hybrid_clock.cc:648] HybridClock initialized: now 1761556449220776 us; error 55 us; skew 500 ppm
09:14:09.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.221698 24948 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:14:09.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 27 09:14:09 dist-test-slave-n41w krb5kdc[24542](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761556449, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.244.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:14:09.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.227643 24948 init.cc:377] Logged in from keytab as kudu/127.23.244.252@KRBTEST.COM (short username kudu)
09:14:09.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.233243 24948 webserver.cc:492] Webserver started at http://127.23.244.252:40373/ using document root <none> and password file <none>
09:14:09.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.234056 24948 fs_manager.cc:362] Metadata directory not provided
09:14:09.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.234262 24948 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:14:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.242115 24948 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.006s sys 0.000s
09:14:09.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.246203 24989 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:14:09.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.247182 24948 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
09:14:09.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.247426 24948 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-2/data,/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-2/wal
09:14:09.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "88fae1c8ea274939a7704e39fcf882f5"
09:14:09.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-27 09:14:02 on dist-test-slave-n41w"
09:14:09.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.249756 24948 fs_report.cc:389] FS layout report
09:14:09.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:14:09.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-2/wal
09:14:09.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-2/wal
09:14:09.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-2/data/data
09:14:09.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:14:09.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:14:09.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:14:09.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:14:09.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:14:09.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:14:09.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:14:09.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:14:09.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:14:09.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:14:09.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:14:09.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.303324 24948 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:14:09.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.308255 24948 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:14:09.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.308624 24948 kserver.cc:163] Server-wide thread pool size limit: 3276
09:14:09.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.384832 24948 rpc_server.cc:307] RPC server started. Bound to: 127.23.244.252:45583
09:14:09.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.384899 25040 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.244.252:45583 every 8 connection(s)
09:14:09.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.387190 24948 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-2/data/info.pb
09:14:09.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.394299 24531 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu as pid 24948
09:14:09.394 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:14:09.394 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.23.244.253:40609
09:14:09.395 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
09:14:09.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.396405 24531 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu
09:14:09.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu
09:14:09.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-1/wal
09:14:09.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-1/data
09:14:09.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:14:09.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:14:09.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:14:09.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:14:09.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:14:09.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:14:09.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-1/logs
09:14:09.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-1/data/info.pb
09:14:09.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:14:09.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:14:09.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:14:09.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:14:09.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:14:09.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:14:09.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:14:09.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:14:09.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:14:09.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:14:09.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:14:09.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:14:09.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.244.253:40609
09:14:09.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.244.253
09:14:09.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45357
09:14:09.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/kudu.keytab
09:14:09.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.244.253
09:14:09.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:14:09.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
09:14:09.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
09:14:09.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
09:14:09.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.244.212:44779
09:14:09.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:14:09.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:14:09.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:14:09.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:14:09.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.244.254:33613,127.23.244.253:40609,127.23.244.252:45583
09:14:09.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:14:09.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38985/jwks.json
09:14:09.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
09:14:09.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
09:14:09.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.397614 25041 sys_catalog.cc:263] Verifying existing consensus state
09:14:09.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.402573 25041 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5: Bootstrap starting.
09:14:09.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.449476 25041 log.cc:826] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5: Log is configured to *not* fsync() on all Append() calls
09:14:09.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.472002 25041 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
09:14:09.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.472842 25041 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5: Bootstrap complete.
09:14:09.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.500236 25041 raft_consensus.cc:359] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c69f67b25f8445fa1bed7d3e83853a0" member_type: VOTER last_known_addr { host: "127.23.244.254" port: 33613 } } peers { permanent_uuid: "4dc4196f0b844bcfbba87180c2307218" member_type: VOTER last_known_addr { host: "127.23.244.253" port: 40609 } } peers { permanent_uuid: "88fae1c8ea274939a7704e39fcf882f5" member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } }
09:14:09.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.502910 25041 raft_consensus.cc:740] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 88fae1c8ea274939a7704e39fcf882f5, State: Initialized, Role: FOLLOWER
09:14:09.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.503722 25041 consensus_queue.cc:260] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c69f67b25f8445fa1bed7d3e83853a0" member_type: VOTER last_known_addr { host: "127.23.244.254" port: 33613 } } peers { permanent_uuid: "4dc4196f0b844bcfbba87180c2307218" member_type: VOTER last_known_addr { host: "127.23.244.253" port: 40609 } } peers { permanent_uuid: "88fae1c8ea274939a7704e39fcf882f5" member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } }
09:14:09.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.507174 25046 sys_catalog.cc:455] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c69f67b25f8445fa1bed7d3e83853a0" member_type: VOTER last_known_addr { host: "127.23.244.254" port: 33613 } } peers { permanent_uuid: "4dc4196f0b844bcfbba87180c2307218" member_type: VOTER last_known_addr { host: "127.23.244.253" port: 40609 } } peers { permanent_uuid: "88fae1c8ea274939a7704e39fcf882f5" member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } } }
09:14:09.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.507961 25046 sys_catalog.cc:458] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5 [sys.catalog]: This master's current role is: FOLLOWER
09:14:09.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.508384 25041 sys_catalog.cc:565] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5 [sys.catalog]: configured and running, proceeding with master startup.
09:14:09.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.545310 25057 catalog_manager.cc:1269] Loaded cluster ID: 34126f028d4542fdb530af64d91fa79a
09:14:09.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.545745 25057 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5: loading cluster ID for follower catalog manager: success
09:14:09.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.552801 25057 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5: acquiring CA information for follower catalog manager: success
09:14:09.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 27 09:14:09 dist-test-slave-n41w krb5kdc[24542](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761556449, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.244.252@KRBTEST.COM for kudu/127.23.244.254@KRBTEST.COM
09:14:09.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.560371 25057 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:14:09.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.575131 24948 master_runner.cc:186] Error getting master registration for 127.23.244.253:40609: Network error: Client connection negotiation failed: client connection to 127.23.244.253:40609: connect: Connection refused (error 111),
09:14:09.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.597967 24948 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
09:14:09.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:09.747033 25043 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:14:09.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:09.747457 25043 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:14:09.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:09.748152 25043 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:14:09.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:09.775199 25043 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
09:14:09.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:09.775403 25043 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:14:09.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:09.775799 25043 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:14:09.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:09.776057 25043 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:14:09.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:09.776300 25043 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:14:09.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:09.776532 25043 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:14:09.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:09.776716 25043 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38985/jwks.json
09:14:09.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.807284 25043 master_runner.cc:386] Master server non-default flags:
09:14:09.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:14:09.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.244.212:44779
09:14:09.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:14:09.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:14:09.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-1/data
09:14:09.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-1/wal
09:14:09.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
09:14:09.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:14:09.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.244.254:33613,127.23.244.253:40609,127.23.244.252:45583
09:14:09.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/krb5kdc/kudu.keytab
09:14:09.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.244.253
09:14:09.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:14:09.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:14:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:14:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.244.253:40609
09:14:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:14:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:14:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:14:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38985/jwks.json
09:14:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:14:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
09:14:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
09:14:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:14:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:14:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-1/data/info.pb
09:14:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.244.253
09:14:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45357
09:14:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:14:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:14:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:14:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:14:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:14:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-1/logs
09:14:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:14:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:14:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:14:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:14:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:14:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d0cb013a5aad754f2ed06c25f5ec3d8a1873622
09:14:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:14:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Oct 2025 08:43:21 UTC on 5fd53c4cbb9d
09:14:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8741
09:14:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:14:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.808665 25043 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:14:09.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:09.809510 25043 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:14:09.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:09.820067 25066 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:14:10.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:10.883157 25074 raft_consensus.cc:493] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:14:10.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:10.883994 25074 raft_consensus.cc:515] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c69f67b25f8445fa1bed7d3e83853a0" member_type: VOTER last_known_addr { host: "127.23.244.254" port: 33613 } } peers { permanent_uuid: "4dc4196f0b844bcfbba87180c2307218" member_type: VOTER last_known_addr { host: "127.23.244.253" port: 40609 } } peers { permanent_uuid: "88fae1c8ea274939a7704e39fcf882f5" member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } }
09:14:10.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:10.892685 24921 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "88fae1c8ea274939a7704e39fcf882f5" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "6c69f67b25f8445fa1bed7d3e83853a0" is_pre_election: true
09:14:10.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:10.893517 24921 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 88fae1c8ea274939a7704e39fcf882f5 in term 1.
09:14:10.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:10.895308 24990 leader_election.cc:304] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6c69f67b25f8445fa1bed7d3e83853a0, 88fae1c8ea274939a7704e39fcf882f5; no voters:
09:14:10.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:10.902722 25075 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5 [term 1 FOLLOWER]: Leader pre-election won for term 2
09:14:10.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:10.903146 25075 raft_consensus.cc:493] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
09:14:10.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:10.903474 25075 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5 [term 1 FOLLOWER]: Advancing to term 2
09:14:10.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:10.910143 25074 leader_election.cc:290] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 6c69f67b25f8445fa1bed7d3e83853a0 (127.23.244.254:33613), 4dc4196f0b844bcfbba87180c2307218 (127.23.244.253:40609)
09:14:10.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:10.922874 24993 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.244.253:40609: connect: Connection refused (error 111)
09:14:10.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:10.928243 25075 raft_consensus.cc:515] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c69f67b25f8445fa1bed7d3e83853a0" member_type: VOTER last_known_addr { host: "127.23.244.254" port: 33613 } } peers { permanent_uuid: "4dc4196f0b844bcfbba87180c2307218" member_type: VOTER last_known_addr { host: "127.23.244.253" port: 40609 } } peers { permanent_uuid: "88fae1c8ea274939a7704e39fcf882f5" member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } }
09:14:10.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:10.932267 24993 leader_election.cc:336] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 4dc4196f0b844bcfbba87180c2307218 (127.23.244.253:40609): Network error: Client connection negotiation failed: client connection to 127.23.244.253:40609: connect: Connection refused (error 111)
09:14:10.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:10.938730 24921 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "88fae1c8ea274939a7704e39fcf882f5" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "6c69f67b25f8445fa1bed7d3e83853a0"
09:14:10.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:10.939383 24921 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0 [term 1 FOLLOWER]: Advancing to term 2
09:14:10.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:10.948387 25075 leader_election.cc:290] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5 [CANDIDATE]: Term 2 election: Requested vote from peers 6c69f67b25f8445fa1bed7d3e83853a0 (127.23.244.254:33613), 4dc4196f0b844bcfbba87180c2307218 (127.23.244.253:40609)
09:14:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:10.952915 24921 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 88fae1c8ea274939a7704e39fcf882f5 in term 2.
09:14:10.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:10.953271 24993 leader_election.cc:336] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 4dc4196f0b844bcfbba87180c2307218 (127.23.244.253:40609): Network error: Client connection negotiation failed: client connection to 127.23.244.253:40609: connect: Connection refused (error 111)
09:14:10.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:10.954144 24990 leader_election.cc:304] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6c69f67b25f8445fa1bed7d3e83853a0, 88fae1c8ea274939a7704e39fcf882f5; no voters: 4dc4196f0b844bcfbba87180c2307218
09:14:10.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:10.955423 25075 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5 [term 2 FOLLOWER]: Leader election won for term 2
09:14:10.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:10.955998 25075 raft_consensus.cc:697] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5 [term 2 LEADER]: Becoming Leader. State: Replica: 88fae1c8ea274939a7704e39fcf882f5, State: Running, Role: LEADER
09:14:10.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:10.957439 25075 consensus_queue.cc:237] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c69f67b25f8445fa1bed7d3e83853a0" member_type: VOTER last_known_addr { host: "127.23.244.254" port: 33613 } } peers { permanent_uuid: "4dc4196f0b844bcfbba87180c2307218" member_type: VOTER last_known_addr { host: "127.23.244.253" port: 40609 } } peers { permanent_uuid: "88fae1c8ea274939a7704e39fcf882f5" member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } }
09:14:10.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:10.979455 25074 sys_catalog.cc:455] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 88fae1c8ea274939a7704e39fcf882f5. Latest consensus state: current_term: 2 leader_uuid: "88fae1c8ea274939a7704e39fcf882f5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c69f67b25f8445fa1bed7d3e83853a0" member_type: VOTER last_known_addr { host: "127.23.244.254" port: 33613 } } peers { permanent_uuid: "4dc4196f0b844bcfbba87180c2307218" member_type: VOTER last_known_addr { host: "127.23.244.253" port: 40609 } } peers { permanent_uuid: "88fae1c8ea274939a7704e39fcf882f5" member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } } }
09:14:10.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:10.980648 25074 sys_catalog.cc:458] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5 [sys.catalog]: This master's current role is: LEADER
09:14:10.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:10.997016 25079 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
09:14:11.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:11.016052 25079 catalog_manager.cc:1494] Initializing Kudu cluster ID...
09:14:11.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:11.032361 25079 catalog_manager.cc:1269] Loaded cluster ID: 34126f028d4542fdb530af64d91fa79a
09:14:11.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:11.032763 25079 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
09:14:11.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:11.039563 25079 catalog_manager.cc:1514] Loading token signing keys...
09:14:11.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:11.050205 25079 catalog_manager.cc:6033] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5: Loaded TSK: 0
09:14:11.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:11.053308 25079 catalog_manager.cc:1524] Initializing in-progress tserver states...
09:14:11.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:11.385916 24921 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0 [term 2 FOLLOWER]: Refusing update from remote peer 88fae1c8ea274939a7704e39fcf882f5: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
09:14:11.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:11.387549 25074 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6c69f67b25f8445fa1bed7d3e83853a0" member_type: VOTER last_known_addr { host: "127.23.244.254" port: 33613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
09:14:11.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:11.412577 25081 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 88fae1c8ea274939a7704e39fcf882f5. Latest consensus state: current_term: 2 leader_uuid: "88fae1c8ea274939a7704e39fcf882f5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c69f67b25f8445fa1bed7d3e83853a0" member_type: VOTER last_known_addr { host: "127.23.244.254" port: 33613 } } peers { permanent_uuid: "4dc4196f0b844bcfbba87180c2307218" member_type: VOTER last_known_addr { host: "127.23.244.253" port: 40609 } } peers { permanent_uuid: "88fae1c8ea274939a7704e39fcf882f5" member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } } }
09:14:11.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:11.413403 25081 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0 [sys.catalog]: This master's current role is: FOLLOWER
09:14:11.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:11.424680 25075 sys_catalog.cc:455] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "88fae1c8ea274939a7704e39fcf882f5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c69f67b25f8445fa1bed7d3e83853a0" member_type: VOTER last_known_addr { host: "127.23.244.254" port: 33613 } } peers { permanent_uuid: "4dc4196f0b844bcfbba87180c2307218" member_type: VOTER last_known_addr { host: "127.23.244.253" port: 40609 } } peers { permanent_uuid: "88fae1c8ea274939a7704e39fcf882f5" member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } } }
09:14:11.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:11.425499 25075 sys_catalog.cc:458] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5 [sys.catalog]: This master's current role is: LEADER
09:14:11.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:11.428299 24993 consensus_peers.cc:597] T 00000000000000000000000000000000 P 88fae1c8ea274939a7704e39fcf882f5 -> Peer 4dc4196f0b844bcfbba87180c2307218 (127.23.244.253:40609): Couldn't send request to peer 4dc4196f0b844bcfbba87180c2307218. Status: Network error: Client connection negotiation failed: client connection to 127.23.244.253:40609: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
09:14:11.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:11.437471 25081 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "88fae1c8ea274939a7704e39fcf882f5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c69f67b25f8445fa1bed7d3e83853a0" member_type: VOTER last_known_addr { host: "127.23.244.254" port: 33613 } } peers { permanent_uuid: "4dc4196f0b844bcfbba87180c2307218" member_type: VOTER last_known_addr { host: "127.23.244.253" port: 40609 } } peers { permanent_uuid: "88fae1c8ea274939a7704e39fcf882f5" member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } } }
09:14:11.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:11.438055 25081 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6c69f67b25f8445fa1bed7d3e83853a0 [sys.catalog]: This master's current role is: FOLLOWER
09:14:11.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:11.222767 25065 debug-util.cc:398] Leaking SignalData structure 0x7b0800027080 after lost signal to thread 25043
09:14:11.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:11.764406 25065 kernel_stack_watchdog.cc:198] Thread 25043 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
09:14:11.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
09:14:11.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
09:14:11.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
09:14:11.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
09:14:11.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:09.821669 25067 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:14:11.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:11.765883 25043 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.946s user 0.660s sys 1.047s
09:14:11.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:11.766209 25043 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.946s user 0.660s sys 1.047s
09:14:11.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:11.767648 25069 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:14:11.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:11.769229 25068 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1949 milliseconds
09:14:11.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:11.769279 25043 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:14:11.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:11.770383 25043 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:14:11.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:11.772600 25043 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:14:11.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:11.773893 25043 hybrid_clock.cc:648] HybridClock initialized: now 1761556451773832 us; error 58 us; skew 500 ppm
09:14:11.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:11.774582 25043 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:14:11.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 27 09:14:11 dist-test-slave-n41w krb5kdc[24542](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761556451, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.244.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:14:11.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:11.779700 25043 init.cc:377] Logged in from keytab as kudu/127.23.244.253@KRBTEST.COM (short username kudu)
09:14:11.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:11.784976 25043 webserver.cc:492] Webserver started at http://127.23.244.253:45357/ using document root <none> and password file <none>
09:14:11.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:11.785751 25043 fs_manager.cc:362] Metadata directory not provided
09:14:11.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:11.785972 25043 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:14:11.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:11.793598 25043 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.002s sys 0.003s
09:14:11.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:11.797709 25090 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:14:11.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:11.798766 25043 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.000s sys 0.004s
09:14:11.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:11.799054 25043 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-1/data,/tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-1/wal
09:14:11.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4dc4196f0b844bcfbba87180c2307218"
09:14:11.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-27 09:13:59 on dist-test-slave-n41w"
09:14:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:11.800810 25043 fs_report.cc:389] FS layout report
09:14:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:14:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-1/wal
09:14:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-1/wal
09:14:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-1/data/data
09:14:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:14:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:14:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:14:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:14:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:14:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:14:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:14:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:14:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:14:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:14:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:14:11.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:11.855867 25043 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:14:11.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:11.862599 25043 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:14:11.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:11.863035 25043 kserver.cc:163] Server-wide thread pool size limit: 3276
09:14:11.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:11.949162 25043 rpc_server.cc:307] RPC server started. Bound to: 127.23.244.253:40609
09:14:11.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:11.949224 25141 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.244.253:40609 every 8 connection(s)
09:14:11.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:11.952282 25043 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskWfs5gY/test-tmp/mini-kudu-cluster4100268366624543974/master-1/data/info.pb
09:14:11.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:11.960400 24531 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu as pid 25043
09:14:11.961 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:14:11.961 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.23.244.254:33613(127.23.244.254:33613)] connecting to peer
09:14:11.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:11.963042 25142 sys_catalog.cc:263] Verifying existing consensus state
09:14:11.963 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.23.244.252:45583(127.23.244.252:45583)] connecting to peer
09:14:11.964 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.23.244.253:40609(127.23.244.253:40609)] connecting to peer
09:14:11.965 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.23.244.254:33613(127.23.244.254:33613)] Successfully connected to peer
09:14:11.973 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.23.244.252:45583(127.23.244.252:45583)] Successfully connected to peer
09:14:11.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:11.973145 25142 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4dc4196f0b844bcfbba87180c2307218: Bootstrap starting.
09:14:11.975 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.23.244.253:40609(127.23.244.253:40609)] Successfully connected to peer
09:14:11.979 [WARN - kudu-nio-0] (SecurityContext.java:206) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
09:14:11.980 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.23.244.254
09:14:11.990 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.23.244.253
09:14:11.991 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.23.244.252
09:14:12.009 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.244.252:45583(127.23.244.252:45583)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
09:14:12.009 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.244.252:45583: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
09:14:12.010 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.23.244.252:45583(127.23.244.252:45583)] handling channelInactive
09:14:12.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:12.020105 25148 negotiation.cc:337] Failed RPC negotiation. Trace:
09:14:12.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1027 09:14:11.971349 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:56080 (local address 127.23.244.252:45583)
09:14:12.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1027 09:14:11.990302 (+ 18953us) server_negotiation.cc:207] Beginning negotiation
09:14:12.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1027 09:14:11.990322 (+ 20us) server_negotiation.cc:400] Waiting for connection header
09:14:12.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1027 09:14:11.990357 (+ 35us) server_negotiation.cc:408] Connection header received
09:14:12.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1027 09:14:11.990470 (+ 113us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
09:14:12.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1027 09:14:11.990496 (+ 26us) server_negotiation.cc:462] Received NEGOTIATE request from client
09:14:12.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1027 09:14:11.990715 (+ 219us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
09:14:12.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1027 09:14:11.990903 (+ 188us) server_negotiation.cc:227] Negotiated authn=SASL
09:14:12.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1027 09:14:11.995660 (+ 4757us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
09:14:12.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1027 09:14:11.997936 (+ 2276us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
09:14:12.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1027 09:14:12.015742 (+ 17806us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
09:14:12.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1027 09:14:12.016292 (+ 550us) server_negotiation.cc:663] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
09:14:12.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1027 09:14:12.018559 (+ 2267us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:56080: BlockingRecv error: failed to read from TLS socket (remote: unknown): Cannot send after transport endpoint shutdown (error 108)
09:14:12.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":18524,"thread_start_us":19722,"threads_started":1}
09:14:12.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:12.020855 25142 log.cc:826] T 00000000000000000000000000000000 P 4dc4196f0b844bcfbba87180c2307218: Log is configured to *not* fsync() on all Append() calls
09:14:12.023 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.244.254:33613(127.23.244.254:33613)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
09:14:12.023 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.244.254:33613: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
09:14:12.024 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.23.244.254:33613(127.23.244.254:33613)] handling channelInactive
09:14:12.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:12.028839 25147 negotiation.cc:337] Failed RPC negotiation. Trace:
09:14:12.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1027 09:14:11.965191 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:57138 (local address 127.23.244.254:33613)
09:14:12.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1027 09:14:11.978115 (+ 12924us) server_negotiation.cc:207] Beginning negotiation
09:14:12.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1027 09:14:11.978141 (+ 26us) server_negotiation.cc:400] Waiting for connection header
09:14:12.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1027 09:14:11.978191 (+ 50us) server_negotiation.cc:408] Connection header received
09:14:12.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1027 09:14:11.978338 (+ 147us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
09:14:12.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1027 09:14:11.978363 (+ 25us) server_negotiation.cc:462] Received NEGOTIATE request from client
09:14:12.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1027 09:14:11.978574 (+ 211us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
09:14:12.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1027 09:14:11.999493 (+ 20919us) server_negotiation.cc:227] Negotiated authn=SASL
09:14:12.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1027 09:14:11.999854 (+ 361us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
09:14:12.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1027 09:14:12.001394 (+ 1540us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
09:14:12.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1027 09:14:12.021886 (+ 20492us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
09:14:12.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1027 09:14:12.022464 (+ 578us) server_negotiation.cc:663] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
09:14:12.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1027 09:14:12.027046 (+ 4582us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:57138: BlockingRecv error: failed to read from TLS socket (remote: unknown): Cannot send after transport endpoint shutdown (error 108)
09:14:12.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":12416,"thread_start_us":13581,"threads_started":1}
09:14:12.035 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.244.253:40609(127.23.244.253:40609)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
09:14:12.035 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.244.253:40609: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
09:14:12.035 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1523) Notify RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=74), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.244.254:33613, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.244.252:45583, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.244.253:40609, [48ms] Sub RPC ConnectToMaster: received response from server master-127.23.244.252:45583: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.244.254:33613: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [74ms] Sub RPC ConnectToMaster: received response from server master-127.23.244.253:40609: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), deferred=null) after lookup exception
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.23.244.254:33613,127.23.244.252:45583,127.23.244.253:40609). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) [main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) [main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) [main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) [main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) [async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) [main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) [main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) [main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) [main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) [main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) [main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) [main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) [main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) [netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.23.244.254:33613,127.23.244.252:45583,127.23.244.253:40609). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
09:14:12.039 [DEBUG - kudu-nio-0] (KuduRpc.java:260) Handling a callback on RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=77), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.244.254:33613, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.244.252:45583, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.244.253:40609, [48ms] Sub RPC ConnectToMaster: received response from server master-127.23.244.252:45583: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.244.254:33613: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [74ms] Sub RPC ConnectToMaster: received response from server master-127.23.244.253:40609: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), deferred=null) with no deferred attached!
09:14:12.039 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.23.244.254:33613(127.23.244.254:33613)] disconnecting while in state TERMINATED
09:14:12.039 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.23.244.252:45583(127.23.244.252:45583)] disconnecting while in state TERMINATED
09:14:12.039 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.23.244.253:40609(127.23.244.253:40609)] handling channelInactive
09:14:12.039 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.23.244.253:40609(127.23.244.253:40609)] disconnecting while in state TERMINATED
09:14:12.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251027 09:14:12.042375 25149 negotiation.cc:337] Failed RPC negotiation. Trace:
09:14:12.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1027 09:14:11.975727 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:50632 (local address 127.23.244.253:40609)
09:14:12.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1027 09:14:11.978249 (+ 2522us) server_negotiation.cc:207] Beginning negotiation
09:14:12.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1027 09:14:11.978283 (+ 34us) server_negotiation.cc:400] Waiting for connection header
09:14:12.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1027 09:14:11.978398 (+ 115us) server_negotiation.cc:408] Connection header received
09:14:12.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1027 09:14:11.984466 (+ 6068us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
09:14:12.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1027 09:14:11.984531 (+ 65us) server_negotiation.cc:462] Received NEGOTIATE request from client
09:14:12.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1027 09:14:11.985025 (+ 494us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
09:14:12.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1027 09:14:11.985500 (+ 475us) server_negotiation.cc:227] Negotiated authn=SASL
09:14:12.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1027 09:14:12.006648 (+ 21148us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
09:14:12.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1027 09:14:12.008970 (+ 2322us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
09:14:12.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1027 09:14:12.034882 (+ 25912us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
09:14:12.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1027 09:14:12.035436 (+ 554us) server_negotiation.cc:663] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
09:14:12.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1027 09:14:12.040621 (+ 5185us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:50632: BlockingRecv error: failed to read from TLS socket (remote: unknown): Cannot send after transport endpoint shutdown (error 108)
09:14:12.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":1833,"thread_start_us":1460,"threads_started":1}
09:14:12.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:12.046751 25142 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4dc4196f0b844bcfbba87180c2307218: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
09:14:12.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:12.047644 25142 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4dc4196f0b844bcfbba87180c2307218: Bootstrap complete.
09:14:12.048 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
09:14:12.049 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
09:14:12.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:12.051808 24531 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu with pid 24877
09:14:12.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:12.077150 25142 raft_consensus.cc:359] T 00000000000000000000000000000000 P 4dc4196f0b844bcfbba87180c2307218 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c69f67b25f8445fa1bed7d3e83853a0" member_type: VOTER last_known_addr { host: "127.23.244.254" port: 33613 } } peers { permanent_uuid: "4dc4196f0b844bcfbba87180c2307218" member_type: VOTER last_known_addr { host: "127.23.244.253" port: 40609 } } peers { permanent_uuid: "88fae1c8ea274939a7704e39fcf882f5" member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } }
09:14:12.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:12.079955 25142 raft_consensus.cc:740] T 00000000000000000000000000000000 P 4dc4196f0b844bcfbba87180c2307218 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4dc4196f0b844bcfbba87180c2307218, State: Initialized, Role: FOLLOWER
09:14:12.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:12.080935 25142 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4dc4196f0b844bcfbba87180c2307218 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c69f67b25f8445fa1bed7d3e83853a0" member_type: VOTER last_known_addr { host: "127.23.244.254" port: 33613 } } peers { permanent_uuid: "4dc4196f0b844bcfbba87180c2307218" member_type: VOTER last_known_addr { host: "127.23.244.253" port: 40609 } } peers { permanent_uuid: "88fae1c8ea274939a7704e39fcf882f5" member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } }
09:14:12.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:12.087157 25155 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4dc4196f0b844bcfbba87180c2307218 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6c69f67b25f8445fa1bed7d3e83853a0" member_type: VOTER last_known_addr { host: "127.23.244.254" port: 33613 } } peers { permanent_uuid: "4dc4196f0b844bcfbba87180c2307218" member_type: VOTER last_known_addr { host: "127.23.244.253" port: 40609 } } peers { permanent_uuid: "88fae1c8ea274939a7704e39fcf882f5" member_type: VOTER last_known_addr { host: "127.23.244.252" port: 45583 } } }
09:14:12.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:12.088020 25155 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4dc4196f0b844bcfbba87180c2307218 [sys.catalog]: This master's current role is: FOLLOWER
09:14:12.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:12.089241 25142 sys_catalog.cc:565] T 00000000000000000000000000000000 P 4dc4196f0b844bcfbba87180c2307218 [sys.catalog]: configured and running, proceeding with master startup.
09:14:12.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:12.090816 24531 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu with pid 25043
09:14:12.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:12.110131 24531 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu with pid 24948
09:14:13.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-27T09:14:13Z chronyd exiting
09:14:14.193 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
09:14:14.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:14.194749 23805 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu with pid 24085
09:14:14.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:14.217120 23805 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu with pid 24241
09:14:14.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:14.238313 23805 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu with pid 24390
09:14:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:14.260002 23805 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu with pid 23817
09:14:14.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:14.288879 23805 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu with pid 23887
09:14:14.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251027 09:14:14.315219 23805 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskWfs5gY/build/tsan/bin/kudu with pid 23962
09:14:15.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-27T09:14:15Z chronyd exiting
09:14:16.342 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectExpires: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.23.244.254:33613,127.23.244.252:45583,127.23.244.253:40609). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386) ~[test/:?]
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40) ~[kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectExpires(TestSecurity.java:412) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.23.244.254:33613,127.23.244.252:45583,127.23.244.253:40609). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386)
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40)
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectExpires(TestSecurity.java:412)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
09:14:16.522 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectExpires: giving up after 1 attempts
09:14:16.522 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectExpires: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.23.244.254:33613,127.23.244.252:45583,127.23.244.253:40609). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386) ~[test/:?]
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40) ~[kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectExpires(TestSecurity.java:412) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.23.244.254:33613,127.23.244.252:45583,127.23.244.253:40609). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386)
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40)
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectExpires(TestSecurity.java:412)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more