01:37:53.285 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
01:37:53.286 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskoFx69z/test-tmp
01:37:53.286 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
01:37:53.286 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskoFx69z/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
01:37:53.287 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
01:37:53.597 [DEBUG - main] (MiniKuduCluster.java:195) Response:
01:37:53.597 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
01:37:53.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:53.599920 21226 mini_oidc.cc:152] Starting JWKS server
01:37:53.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:53.603250 21226 webserver.cc:460] Webserver started at http://127.0.0.1:38329/ using document root <none> and password file <none>
01:37:53.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:53.603629 21226 mini_oidc.cc:189] Starting OIDC Discovery server
01:37:53.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:53.605343 21226 webserver.cc:460] Webserver started at http://127.0.0.1:35283/ using document root <none> and password file <none>
01:37:53.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T01:37:53Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
01:37:53.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T01:37:53Z Disabled control of system clock
01:37:53.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:53.653838 21226 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu
01:37:53.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu
01:37:53.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-0/wal
01:37:53.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-0/data
01:37:53.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:37:53.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:37:53.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:37:53.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:37:53.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:37:53.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:37:53.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-0/logs
01:37:53.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-0/data/info.pb
01:37:53.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:37:53.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:37:53.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:37:53.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:37:53.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:37:53.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:37:53.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:37:53.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:37:53.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:37:53.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:37:53.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:37:53.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:37:53.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.186.190:35505
01:37:53.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.186.190
01:37:53.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:37:53.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.186.148:33549
01:37:53.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:37:53.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:37:53.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:37:53.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:37:53.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.186.190:35505,127.20.186.189:38631,127.20.186.188:40445
01:37:53.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:37:53.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38329/jwks.json with env {}
01:37:53.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:53.955055 21238 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:37:53.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:53.955685 21238 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:37:53.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:53.956254 21238 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:37:53.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:53.986450 21238 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:37:53.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:53.986827 21238 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:37:53.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:53.987131 21238 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:37:53.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:53.987443 21238 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:37:53.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:53.987771 21238 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:37:53.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:53.987989 21238 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38329/jwks.json
01:37:54.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:54.021508 21238 master_runner.cc:386] Master server non-default flags:
01:37:54.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:37:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.186.148:33549
01:37:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:37:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:37:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-0/data
01:37:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-0/wal
01:37:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:37:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.186.190:35505,127.20.186.189:38631,127.20.186.188:40445
01:37:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:37:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:37:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:37:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.186.190:35505
01:37:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:37:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:37:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:37:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38329/jwks.json
01:37:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:37:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:37:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-0/data/info.pb
01:37:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.186.190
01:37:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:37:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:37:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:37:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:37:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:37:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:37:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-0/logs
01:37:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:37:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:37:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:37:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:37:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:37:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:37:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:37:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 00:02:55 UTC on ccycloud.quasar-qgnmee.root.comops.site
01:37:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675483
01:37:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:37:54.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:54.022984 21238 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:37:54.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:54.024502 21238 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:37:54.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:54.041299 21245 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:37:55.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:54.041688 21247 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:37:55.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:54.042028 21244 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:37:55.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:54.042542 21238 server_base.cc:1034] running on GCE node
01:37:55.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:55.227372 21238 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:37:55.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:55.230196 21238 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:37:55.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:55.231567 21238 hybrid_clock.cc:648] HybridClock initialized: now 1790473075231518 us; error 57 us; skew 500 ppm
01:37:55.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:55.232445 21238 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:37:55.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:55.238651 21238 webserver.cc:460] Webserver started at http://127.20.186.190:36919/ using document root <none> and password file <none>
01:37:55.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:55.239598 21238 fs_manager.cc:359] Metadata directory not provided
01:37:55.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:55.239851 21238 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:37:55.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:55.240417 21238 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:37:55.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:55.245241 21238 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-0/data/instance:
01:37:55.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "720cceaedd1448b39b341feb871496ae"
01:37:55.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 01:37:55 on dist-test-slave-37dq"
01:37:55.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:55.246248 21238 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-0/wal/instance:
01:37:55.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "720cceaedd1448b39b341feb871496ae"
01:37:55.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 01:37:55 on dist-test-slave-37dq"
01:37:55.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:55.252830 21238 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.007s sys 0.001s
01:37:55.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:55.257963 21254 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:37:55.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:55.258977 21238 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.003s
01:37:55.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:55.259238 21238 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-0/data,/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-0/wal
01:37:55.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "720cceaedd1448b39b341feb871496ae"
01:37:55.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 01:37:55 on dist-test-slave-37dq"
01:37:55.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:55.259507 21238 fs_report.cc:383] FS layout report
01:37:55.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:37:55.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-0/wal
01:37:55.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-0/wal
01:37:55.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-0/data/data
01:37:55.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:37:55.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:37:55.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:37:55.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:37:55.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:37:55.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:37:55.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:37:55.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:37:55.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:37:55.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:37:55.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:37:55.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:55.310968 21238 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:37:55.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:55.312382 21238 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:37:55.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:55.312836 21238 kserver.cc:163] Server-wide thread pool size limit: 3276
01:37:55.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:55.380717 21238 rpc_server.cc:307] RPC server started. Bound to: 127.20.186.190:35505
01:37:55.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:55.380832 21305 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.186.190:35505 every 8 connection(s)
01:37:55.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:55.383373 21238 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-0/data/info.pb
01:37:55.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:55.387804 21226 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu as pid 21238
01:37:55.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:55.388262 21226 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-0/wal/instance
01:37:55.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:55.389663 21306 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:37:55.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:55.391610 21226 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu
01:37:55.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu
01:37:55.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-1/wal
01:37:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-1/data
01:37:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:37:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:37:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:37:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:37:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:37:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:37:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-1/logs
01:37:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-1/data/info.pb
01:37:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:37:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:37:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:37:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:37:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:37:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:37:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:37:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:37:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:37:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:37:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:37:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:37:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.186.189:38631
01:37:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.186.189
01:37:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:37:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.186.148:33549
01:37:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:37:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:37:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:37:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:37:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.186.190:35505,127.20.186.189:38631,127.20.186.188:40445
01:37:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:37:55.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38329/jwks.json with env {}
01:37:55.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:55.401281 21306 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.186.190" port: 35505 } has no permanent_uuid. Determining permanent_uuid...
01:37:55.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:55.428220 21306 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.186.189" port: 38631 } has no permanent_uuid. Determining permanent_uuid...
01:37:55.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:55.430547 21257 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.186.189:38631: connect: Connection refused (error 111)
01:37:55.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:55.434402 21306 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.186.189:38631: Network error: Client connection negotiation failed: client connection to 127.20.186.189:38631: connect: Connection refused (error 111)
01:37:55.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:55.483774 21306 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.186.189" port: 38631 } attempt: 1
01:37:55.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:55.487430 21306 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.186.189:38631: Network error: Client connection negotiation failed: client connection to 127.20.186.189:38631: connect: Connection refused (error 111)
01:37:55.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:55.555887 21306 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.186.189" port: 38631 } attempt: 2
01:37:55.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:55.559579 21306 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.186.189:38631: Network error: Client connection negotiation failed: client connection to 127.20.186.189:38631: connect: Connection refused (error 111)
01:37:55.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:55.651049 21306 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.186.189" port: 38631 } attempt: 3
01:37:55.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:55.654867 21306 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.186.189:38631: Network error: Client connection negotiation failed: client connection to 127.20.186.189:38631: connect: Connection refused (error 111)
01:37:55.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:55.690111 21308 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:37:55.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:55.690709 21308 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:37:55.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:55.691288 21308 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:37:55.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:55.721035 21308 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:37:55.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:55.721392 21308 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:37:55.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:55.721698 21308 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:37:55.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:55.721989 21308 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:37:55.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:55.722285 21308 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:37:55.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:55.722527 21308 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38329/jwks.json
01:37:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:55.755811 21308 master_runner.cc:386] Master server non-default flags:
01:37:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:37:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.186.148:33549
01:37:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:37:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:37:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-1/data
01:37:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-1/wal
01:37:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:37:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.186.190:35505,127.20.186.189:38631,127.20.186.188:40445
01:37:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:37:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:37:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:37:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.186.189:38631
01:37:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:37:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:37:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:37:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38329/jwks.json
01:37:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:37:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:37:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-1/data/info.pb
01:37:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.186.189
01:37:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:37:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:37:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:37:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:37:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:37:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:37:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-1/logs
01:37:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:37:55.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:37:55.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:37:55.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:37:55.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:37:55.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:37:55.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:37:55.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 00:02:55 UTC on ccycloud.quasar-qgnmee.root.comops.site
01:37:55.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675483
01:37:55.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:37:55.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:55.757288 21308 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:37:55.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:55.758834 21308 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:37:55.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:55.776396 21317 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:37:55.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:55.798317 21306 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.186.189" port: 38631 } attempt: 4
01:37:55.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:55.801712 21306 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.186.189:38631: Network error: Client connection negotiation failed: client connection to 127.20.186.189:38631: connect: Connection refused (error 111)
01:37:56.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:56.101322 21306 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.186.189" port: 38631 } attempt: 5
01:37:56.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:56.105263 21306 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.186.189:38631: Network error: Client connection negotiation failed: client connection to 127.20.186.189:38631: connect: Connection refused (error 111)
01:37:56.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:56.652833 21306 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.186.189" port: 38631 } attempt: 6
01:37:56.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:56.659757 21306 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.186.189:38631: Network error: Client connection negotiation failed: client connection to 127.20.186.189:38631: connect: Connection refused (error 111)
01:37:56.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:55.776467 21320 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:37:56.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:55.778144 21308 server_base.cc:1034] running on GCE node
01:37:56.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:55.776490 21318 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:37:56.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:56.953522 21308 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:37:56.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:56.956636 21308 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:37:56.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:56.958021 21308 hybrid_clock.cc:648] HybridClock initialized: now 1790473076957987 us; error 59 us; skew 500 ppm
01:37:56.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:56.958801 21308 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:37:56.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:56.964915 21308 webserver.cc:460] Webserver started at http://127.20.186.189:33429/ using document root <none> and password file <none>
01:37:56.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:56.965745 21308 fs_manager.cc:359] Metadata directory not provided
01:37:56.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:56.965910 21308 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:37:56.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:56.966266 21308 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:37:56.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:56.971004 21308 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-1/data/instance:
01:37:56.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8c0053845799434e955627d7f5ad46de"
01:37:56.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 01:37:56 on dist-test-slave-37dq"
01:37:56.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:56.972373 21308 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-1/wal/instance:
01:37:56.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8c0053845799434e955627d7f5ad46de"
01:37:56.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 01:37:56 on dist-test-slave-37dq"
01:37:56.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:56.979303 21308 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.005s sys 0.000s
01:37:56.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:56.984714 21329 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:37:56.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:56.985713 21308 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.001s sys 0.003s
01:37:56.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:56.985983 21308 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-1/data,/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-1/wal
01:37:56.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8c0053845799434e955627d7f5ad46de"
01:37:56.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 01:37:56 on dist-test-slave-37dq"
01:37:56.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:56.986263 21308 fs_report.cc:383] FS layout report
01:37:56.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:37:56.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-1/wal
01:37:56.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-1/wal
01:37:56.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-1/data/data
01:37:56.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:37:56.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:37:56.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:37:56.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:37:56.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:37:56.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:37:56.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:37:56.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:37:56.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:37:56.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:37:56.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:37:57.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:57.015724 21308 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:37:57.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:57.017182 21308 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:37:57.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:57.017567 21308 kserver.cc:163] Server-wide thread pool size limit: 3276
01:37:57.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:57.086503 21308 rpc_server.cc:307] RPC server started. Bound to: 127.20.186.189:38631
01:37:57.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:57.086598 21380 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.186.189:38631 every 8 connection(s)
01:37:57.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:57.089061 21308 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-1/data/info.pb
01:37:57.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:57.093825 21381 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:37:57.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:57.097702 21226 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu as pid 21308
01:37:57.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:57.098088 21226 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-1/wal/instance
01:37:57.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:57.099964 21226 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu
01:37:57.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu
01:37:57.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-2/wal
01:37:57.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-2/data
01:37:57.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:37:57.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:37:57.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:37:57.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:37:57.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:37:57.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:37:57.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-2/logs
01:37:57.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-2/data/info.pb
01:37:57.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:37:57.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:37:57.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:37:57.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:37:57.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:37:57.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:37:57.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:37:57.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:37:57.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:37:57.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:37:57.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:37:57.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:37:57.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.186.188:40445
01:37:57.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.186.188
01:37:57.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:37:57.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.186.148:33549
01:37:57.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:37:57.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:37:57.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:37:57.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:37:57.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.186.190:35505,127.20.186.189:38631,127.20.186.188:40445
01:37:57.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:37:57.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38329/jwks.json with env {}
01:37:57.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:57.104236 21381 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.186.190" port: 35505 } has no permanent_uuid. Determining permanent_uuid...
01:37:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:57.125780 21381 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.186.189" port: 38631 } has no permanent_uuid. Determining permanent_uuid...
01:37:57.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:57.141891 21381 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.186.188" port: 40445 } has no permanent_uuid. Determining permanent_uuid...
01:37:57.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:57.144060 21330 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.186.188:40445: connect: Connection refused (error 111)
01:37:57.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:57.148343 21381 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.186.188:40445: Network error: Client connection negotiation failed: client connection to 127.20.186.188:40445: connect: Connection refused (error 111)
01:37:57.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:57.197865 21381 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.186.188" port: 40445 } attempt: 1
01:37:57.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:57.201843 21381 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.186.188:40445: Network error: Client connection negotiation failed: client connection to 127.20.186.188:40445: connect: Connection refused (error 111)
01:37:57.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:57.270242 21381 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.186.188" port: 40445 } attempt: 2
01:37:57.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:57.274371 21381 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.186.188:40445: Network error: Client connection negotiation failed: client connection to 127.20.186.188:40445: connect: Connection refused (error 111)
01:37:57.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:57.365818 21381 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.186.188" port: 40445 } attempt: 3
01:37:57.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:57.369750 21381 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.186.188:40445: Network error: Client connection negotiation failed: client connection to 127.20.186.188:40445: connect: Connection refused (error 111)
01:37:57.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:57.398083 21383 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:37:57.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:57.398605 21383 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:37:57.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:57.399052 21383 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:37:57.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:57.428858 21383 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:37:57.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:57.429116 21383 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:37:57.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:57.429309 21383 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:37:57.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:57.429507 21383 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:37:57.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:57.429689 21383 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:37:57.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:57.429828 21383 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38329/jwks.json
01:37:57.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:57.463491 21383 master_runner.cc:386] Master server non-default flags:
01:37:57.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:37:57.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.186.148:33549
01:37:57.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:37:57.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:37:57.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-2/data
01:37:57.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-2/wal
01:37:57.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:37:57.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.186.190:35505,127.20.186.189:38631,127.20.186.188:40445
01:37:57.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:37:57.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:37:57.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:37:57.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.186.188:40445
01:37:57.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:37:57.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:37:57.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:37:57.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38329/jwks.json
01:37:57.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:37:57.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:37:57.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-2/data/info.pb
01:37:57.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.186.188
01:37:57.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:37:57.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:37:57.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:37:57.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:37:57.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:37:57.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:37:57.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-2/logs
01:37:57.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:37:57.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:37:57.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:37:57.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:37:57.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:37:57.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:37:57.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:37:57.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 00:02:55 UTC on ccycloud.quasar-qgnmee.root.comops.site
01:37:57.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675483
01:37:57.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:37:57.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:57.464629 21383 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:37:57.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:57.466068 21383 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:37:57.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:57.482774 21393 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:37:57.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:57.513147 21381 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.186.188" port: 40445 } attempt: 4
01:37:57.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:57.516610 21381 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.186.188:40445: Network error: Client connection negotiation failed: client connection to 127.20.186.188:40445: connect: Connection refused (error 111)
01:37:57.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:57.720359 21306 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.186.189" port: 38631 } attempt: 7
01:37:57.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:57.752581 21306 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.186.188" port: 40445 } has no permanent_uuid. Determining permanent_uuid...
01:37:57.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:57.759871 21306 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.186.188:40445: Network error: Client connection negotiation failed: client connection to 127.20.186.188:40445: connect: Connection refused (error 111)
01:37:57.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:57.816025 21381 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.186.188" port: 40445 } attempt: 5
01:37:57.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:57.818464 21306 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.186.188" port: 40445 } attempt: 1
01:37:57.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:57.822674 21381 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.186.188:40445: Network error: Client connection negotiation failed: client connection to 127.20.186.188:40445: connect: Connection refused (error 111)
01:37:57.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:57.824724 21306 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.186.188:40445: Network error: Client connection negotiation failed: client connection to 127.20.186.188:40445: connect: Connection refused (error 111)
01:37:57.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:57.906281 21306 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.186.188" port: 40445 } attempt: 2
01:37:57.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:57.911902 21306 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.186.188:40445: Network error: Client connection negotiation failed: client connection to 127.20.186.188:40445: connect: Connection refused (error 111)
01:37:57.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:57.997457 21306 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.186.188" port: 40445 } attempt: 3
01:37:58.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:58.003041 21306 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.186.188:40445: Network error: Client connection negotiation failed: client connection to 127.20.186.188:40445: connect: Connection refused (error 111)
01:37:58.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:58.143639 21306 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.186.188" port: 40445 } attempt: 4
01:37:58.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:58.149412 21306 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.186.188:40445: Network error: Client connection negotiation failed: client connection to 127.20.186.188:40445: connect: Connection refused (error 111)
01:37:58.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:58.370144 21381 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.186.188" port: 40445 } attempt: 6
01:37:58.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:58.381479 21381 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.186.188:40445: Network error: Client connection negotiation failed: client connection to 127.20.186.188:40445: connect: Connection refused (error 111)
01:37:58.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:58.432956 21306 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.186.188" port: 40445 } attempt: 5
01:37:58.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:58.438712 21306 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.186.188:40445: Network error: Client connection negotiation failed: client connection to 127.20.186.188:40445: connect: Connection refused (error 111)
01:37:58.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:57.482822 21396 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:37:58.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:57.482947 21394 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:37:58.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:58.728852 21395 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1244 milliseconds
01:37:58.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:58.728999 21383 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:37:58.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:58.730127 21383 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:37:58.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:58.733114 21383 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:37:58.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:58.734527 21383 hybrid_clock.cc:648] HybridClock initialized: now 1790473078734480 us; error 65 us; skew 500 ppm
01:37:58.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:58.735390 21383 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:37:58.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:58.741504 21383 webserver.cc:460] Webserver started at http://127.20.186.188:44867/ using document root <none> and password file <none>
01:37:58.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:58.742518 21383 fs_manager.cc:359] Metadata directory not provided
01:37:58.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:58.742736 21383 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:37:58.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:58.743249 21383 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:37:58.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:58.747834 21383 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-2/data/instance:
01:37:58.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "131b1371939e4d14ac8b8dfa9638fbf6"
01:37:58.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 01:37:58 on dist-test-slave-37dq"
01:37:58.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:58.749001 21383 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-2/wal/instance:
01:37:58.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "131b1371939e4d14ac8b8dfa9638fbf6"
01:37:58.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 01:37:58 on dist-test-slave-37dq"
01:37:58.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:58.755553 21383 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.005s sys 0.000s
01:37:58.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:58.760810 21408 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:37:58.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:58.761710 21383 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
01:37:58.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:58.762131 21383 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-2/data,/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-2/wal
01:37:58.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "131b1371939e4d14ac8b8dfa9638fbf6"
01:37:58.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 01:37:58 on dist-test-slave-37dq"
01:37:58.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:58.762459 21383 fs_report.cc:383] FS layout report
01:37:58.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:37:58.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-2/wal
01:37:58.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-2/wal
01:37:58.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-2/data/data
01:37:58.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:37:58.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:37:58.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:37:58.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:37:58.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:37:58.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:37:58.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:37:58.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:37:58.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:37:58.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:37:58.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:37:58.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:58.797786 21383 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:37:58.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:58.799264 21383 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:37:58.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:58.799742 21383 kserver.cc:163] Server-wide thread pool size limit: 3276
01:37:58.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:58.867913 21383 rpc_server.cc:307] RPC server started. Bound to: 127.20.186.188:40445
01:37:58.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:58.868021 21459 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.186.188:40445 every 8 connection(s)
01:37:58.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:58.870751 21383 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-2/data/info.pb
01:37:58.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:58.872421 21226 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu as pid 21383
01:37:58.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:58.872987 21226 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/master-2/wal/instance
01:37:58.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:58.876605 21460 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:37:58.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:58.892234 21460 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.186.190" port: 35505 } has no permanent_uuid. Determining permanent_uuid...
01:37:58.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:58.918365 21460 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.186.189" port: 38631 } has no permanent_uuid. Determining permanent_uuid...
01:37:58.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:58.928912 21460 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.186.188" port: 40445 } has no permanent_uuid. Determining permanent_uuid...
01:37:58.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:58.945293 21460 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 131b1371939e4d14ac8b8dfa9638fbf6: Bootstrap starting.
01:37:58.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:58.950762 21460 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 131b1371939e4d14ac8b8dfa9638fbf6: Neither blocks nor log segments found. Creating new log.
01:37:58.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:58.952987 21460 log.cc:824] T 00000000000000000000000000000000 P 131b1371939e4d14ac8b8dfa9638fbf6: Log is configured to *not* fsync() on all Append() calls
01:37:58.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:58.957124 21460 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 131b1371939e4d14ac8b8dfa9638fbf6: No bootstrap required, opened a new log
01:37:58.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:58.978618 21460 raft_consensus.cc:348] T 00000000000000000000000000000000 P 131b1371939e4d14ac8b8dfa9638fbf6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "720cceaedd1448b39b341feb871496ae" member_type: VOTER last_known_addr { host: "127.20.186.190" port: 35505 } } peers { permanent_uuid: "8c0053845799434e955627d7f5ad46de" member_type: VOTER last_known_addr { host: "127.20.186.189" port: 38631 } } peers { permanent_uuid: "131b1371939e4d14ac8b8dfa9638fbf6" member_type: VOTER last_known_addr { host: "127.20.186.188" port: 40445 } }
01:37:58.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:58.979230 21460 raft_consensus.cc:374] T 00000000000000000000000000000000 P 131b1371939e4d14ac8b8dfa9638fbf6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:37:58.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:58.979447 21460 raft_consensus.cc:729] T 00000000000000000000000000000000 P 131b1371939e4d14ac8b8dfa9638fbf6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 131b1371939e4d14ac8b8dfa9638fbf6, State: Initialized, Role: FOLLOWER
01:37:58.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:58.980196 21460 consensus_queue.cc:260] T 00000000000000000000000000000000 P 131b1371939e4d14ac8b8dfa9638fbf6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "720cceaedd1448b39b341feb871496ae" member_type: VOTER last_known_addr { host: "127.20.186.190" port: 35505 } } peers { permanent_uuid: "8c0053845799434e955627d7f5ad46de" member_type: VOTER last_known_addr { host: "127.20.186.189" port: 38631 } } peers { permanent_uuid: "131b1371939e4d14ac8b8dfa9638fbf6" member_type: VOTER last_known_addr { host: "127.20.186.188" port: 40445 } }
01:37:58.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:58.982995 21469 sys_catalog.cc:455] T 00000000000000000000000000000000 P 131b1371939e4d14ac8b8dfa9638fbf6 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "720cceaedd1448b39b341feb871496ae" member_type: VOTER last_known_addr { host: "127.20.186.190" port: 35505 } } peers { permanent_uuid: "8c0053845799434e955627d7f5ad46de" member_type: VOTER last_known_addr { host: "127.20.186.189" port: 38631 } } peers { permanent_uuid: "131b1371939e4d14ac8b8dfa9638fbf6" member_type: VOTER last_known_addr { host: "127.20.186.188" port: 40445 } } }
01:37:58.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:58.983732 21469 sys_catalog.cc:458] T 00000000000000000000000000000000 P 131b1371939e4d14ac8b8dfa9638fbf6 [sys.catalog]: This master's current role is: FOLLOWER
01:37:58.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:58.985060 21460 sys_catalog.cc:564] T 00000000000000000000000000000000 P 131b1371939e4d14ac8b8dfa9638fbf6 [sys.catalog]: configured and running, proceeding with master startup.
01:37:58.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:58.991268 21306 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.186.188" port: 40445 } attempt: 6
01:37:59.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:59.013556 21481 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 131b1371939e4d14ac8b8dfa9638fbf6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:37:59.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:59.014102 21481 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:37:59.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.015065 21306 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 720cceaedd1448b39b341feb871496ae: Bootstrap starting.
01:37:59.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.025111 21306 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 720cceaedd1448b39b341feb871496ae: Neither blocks nor log segments found. Creating new log.
01:37:59.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.027475 21306 log.cc:824] T 00000000000000000000000000000000 P 720cceaedd1448b39b341feb871496ae: Log is configured to *not* fsync() on all Append() calls
01:37:59.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.032500 21306 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 720cceaedd1448b39b341feb871496ae: No bootstrap required, opened a new log
01:37:59.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.007661 21383 master_runner.cc:186] Error getting master registration for 127.20.186.190:35505: OK, instance_id { permanent_uuid: "720cceaedd1448b39b341feb871496ae" instance_seqno: 1790473075361854 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
01:37:59.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.035180 21383 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:37:59.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:59.054859 21414 tablet.cc:2369] T 00000000000000000000000000000000 P 131b1371939e4d14ac8b8dfa9638fbf6: Can't schedule compaction. Clean time has not been advanced past its initial value.
01:37:59.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.062880 21306 raft_consensus.cc:348] T 00000000000000000000000000000000 P 720cceaedd1448b39b341feb871496ae [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "720cceaedd1448b39b341feb871496ae" member_type: VOTER last_known_addr { host: "127.20.186.190" port: 35505 } } peers { permanent_uuid: "8c0053845799434e955627d7f5ad46de" member_type: VOTER last_known_addr { host: "127.20.186.189" port: 38631 } } peers { permanent_uuid: "131b1371939e4d14ac8b8dfa9638fbf6" member_type: VOTER last_known_addr { host: "127.20.186.188" port: 40445 } }
01:37:59.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.063716 21306 raft_consensus.cc:374] T 00000000000000000000000000000000 P 720cceaedd1448b39b341feb871496ae [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:37:59.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.064014 21306 raft_consensus.cc:729] T 00000000000000000000000000000000 P 720cceaedd1448b39b341feb871496ae [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 720cceaedd1448b39b341feb871496ae, State: Initialized, Role: FOLLOWER
01:37:59.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.064922 21306 consensus_queue.cc:260] T 00000000000000000000000000000000 P 720cceaedd1448b39b341feb871496ae [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "720cceaedd1448b39b341feb871496ae" member_type: VOTER last_known_addr { host: "127.20.186.190" port: 35505 } } peers { permanent_uuid: "8c0053845799434e955627d7f5ad46de" member_type: VOTER last_known_addr { host: "127.20.186.189" port: 38631 } } peers { permanent_uuid: "131b1371939e4d14ac8b8dfa9638fbf6" member_type: VOTER last_known_addr { host: "127.20.186.188" port: 40445 } }
01:37:59.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.067692 21484 sys_catalog.cc:455] T 00000000000000000000000000000000 P 720cceaedd1448b39b341feb871496ae [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "720cceaedd1448b39b341feb871496ae" member_type: VOTER last_known_addr { host: "127.20.186.190" port: 35505 } } peers { permanent_uuid: "8c0053845799434e955627d7f5ad46de" member_type: VOTER last_known_addr { host: "127.20.186.189" port: 38631 } } peers { permanent_uuid: "131b1371939e4d14ac8b8dfa9638fbf6" member_type: VOTER last_known_addr { host: "127.20.186.188" port: 40445 } } }
01:37:59.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.068349 21484 sys_catalog.cc:458] T 00000000000000000000000000000000 P 720cceaedd1448b39b341feb871496ae [sys.catalog]: This master's current role is: FOLLOWER
01:37:59.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.069119 21484 raft_consensus.cc:482] T 00000000000000000000000000000000 P 720cceaedd1448b39b341feb871496ae [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:37:59.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.069659 21306 sys_catalog.cc:564] T 00000000000000000000000000000000 P 720cceaedd1448b39b341feb871496ae [sys.catalog]: configured and running, proceeding with master startup.
01:37:59.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.069504 21484 raft_consensus.cc:504] T 00000000000000000000000000000000 P 720cceaedd1448b39b341feb871496ae [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "720cceaedd1448b39b341feb871496ae" member_type: VOTER last_known_addr { host: "127.20.186.190" port: 35505 } } peers { permanent_uuid: "8c0053845799434e955627d7f5ad46de" member_type: VOTER last_known_addr { host: "127.20.186.189" port: 38631 } } peers { permanent_uuid: "131b1371939e4d14ac8b8dfa9638fbf6" member_type: VOTER last_known_addr { host: "127.20.186.188" port: 40445 } }
01:37:59.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:59.071461 21260 tablet.cc:2369] T 00000000000000000000000000000000 P 720cceaedd1448b39b341feb871496ae: Can't schedule compaction. Clean time has not been advanced past its initial value.
01:37:59.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.074249 21435 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "720cceaedd1448b39b341feb871496ae" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "131b1371939e4d14ac8b8dfa9638fbf6" is_pre_election: true
01:37:59.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.075184 21435 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 131b1371939e4d14ac8b8dfa9638fbf6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 720cceaedd1448b39b341feb871496ae in term 0.
01:37:59.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.076944 21255 leader_election.cc:302] T 00000000000000000000000000000000 P 720cceaedd1448b39b341feb871496ae [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 131b1371939e4d14ac8b8dfa9638fbf6, 720cceaedd1448b39b341feb871496ae; no voters:
01:37:59.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.082676 21484 leader_election.cc:288] T 00000000000000000000000000000000 P 720cceaedd1448b39b341feb871496ae [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8c0053845799434e955627d7f5ad46de (127.20.186.189:38631), 131b1371939e4d14ac8b8dfa9638fbf6 (127.20.186.188:40445)
01:37:59.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.083164 21484 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 720cceaedd1448b39b341feb871496ae [term 0 FOLLOWER]: Leader pre-election won for term 1
01:37:59.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.083478 21484 raft_consensus.cc:482] T 00000000000000000000000000000000 P 720cceaedd1448b39b341feb871496ae [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
01:37:59.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.084877 21484 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 720cceaedd1448b39b341feb871496ae [term 0 FOLLOWER]: Advancing to term 1
01:37:59.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.094986 21484 raft_consensus.cc:504] T 00000000000000000000000000000000 P 720cceaedd1448b39b341feb871496ae [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "720cceaedd1448b39b341feb871496ae" member_type: VOTER last_known_addr { host: "127.20.186.190" port: 35505 } } peers { permanent_uuid: "8c0053845799434e955627d7f5ad46de" member_type: VOTER last_known_addr { host: "127.20.186.189" port: 38631 } } peers { permanent_uuid: "131b1371939e4d14ac8b8dfa9638fbf6" member_type: VOTER last_known_addr { host: "127.20.186.188" port: 40445 } }
01:37:59.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.098049 21484 leader_election.cc:288] T 00000000000000000000000000000000 P 720cceaedd1448b39b341feb871496ae [CANDIDATE]: Term 1 election: Requested vote from peers 8c0053845799434e955627d7f5ad46de (127.20.186.189:38631), 131b1371939e4d14ac8b8dfa9638fbf6 (127.20.186.188:40445)
01:37:59.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.098692 21435 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "720cceaedd1448b39b341feb871496ae" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "131b1371939e4d14ac8b8dfa9638fbf6"
01:37:59.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.099248 21435 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 131b1371939e4d14ac8b8dfa9638fbf6 [term 0 FOLLOWER]: Advancing to term 1
01:37:59.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.112972 21435 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 131b1371939e4d14ac8b8dfa9638fbf6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 720cceaedd1448b39b341feb871496ae in term 1.
01:37:59.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.114881 21255 leader_election.cc:302] T 00000000000000000000000000000000 P 720cceaedd1448b39b341feb871496ae [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 131b1371939e4d14ac8b8dfa9638fbf6, 720cceaedd1448b39b341feb871496ae; no voters:
01:37:59.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.115794 21484 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 720cceaedd1448b39b341feb871496ae [term 1 FOLLOWER]: Leader election won for term 1
01:37:59.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.116235 21484 raft_consensus.cc:686] T 00000000000000000000000000000000 P 720cceaedd1448b39b341feb871496ae [term 1 LEADER]: Becoming Leader. State: Replica: 720cceaedd1448b39b341feb871496ae, State: Running, Role: LEADER
01:37:59.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.117141 21484 consensus_queue.cc:237] T 00000000000000000000000000000000 P 720cceaedd1448b39b341feb871496ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "720cceaedd1448b39b341feb871496ae" member_type: VOTER last_known_addr { host: "127.20.186.190" port: 35505 } } peers { permanent_uuid: "8c0053845799434e955627d7f5ad46de" member_type: VOTER last_known_addr { host: "127.20.186.189" port: 38631 } } peers { permanent_uuid: "131b1371939e4d14ac8b8dfa9638fbf6" member_type: VOTER last_known_addr { host: "127.20.186.188" port: 40445 } }
01:37:59.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.133383 21486 sys_catalog.cc:455] T 00000000000000000000000000000000 P 720cceaedd1448b39b341feb871496ae [sys.catalog]: SysCatalogTable state changed. Reason: New leader 720cceaedd1448b39b341feb871496ae. Latest consensus state: current_term: 1 leader_uuid: "720cceaedd1448b39b341feb871496ae" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "720cceaedd1448b39b341feb871496ae" member_type: VOTER last_known_addr { host: "127.20.186.190" port: 35505 } } peers { permanent_uuid: "8c0053845799434e955627d7f5ad46de" member_type: VOTER last_known_addr { host: "127.20.186.189" port: 38631 } } peers { permanent_uuid: "131b1371939e4d14ac8b8dfa9638fbf6" member_type: VOTER last_known_addr { host: "127.20.186.188" port: 40445 } } }
01:37:59.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.134205 21486 sys_catalog.cc:458] T 00000000000000000000000000000000 P 720cceaedd1448b39b341feb871496ae [sys.catalog]: This master's current role is: LEADER
01:37:59.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:59.163619 21498 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 720cceaedd1448b39b341feb871496ae: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:37:59.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:59.163935 21498 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:37:59.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.164343 21493 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
01:37:59.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.171623 21493 catalog_manager.cc:1484] Initializing Kudu cluster ID...
01:37:59.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.161564 21238 master_runner.cc:186] Error getting master registration for 127.20.186.189:38631: OK, instance_id { permanent_uuid: "8c0053845799434e955627d7f5ad46de" instance_seqno: 1790473077067596 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
01:37:59.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.217054 21238 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:37:59.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.224296 21435 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 131b1371939e4d14ac8b8dfa9638fbf6 [term 1 FOLLOWER]: Refusing update from remote peer 720cceaedd1448b39b341feb871496ae: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
01:37:59.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.232043 21486 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 720cceaedd1448b39b341feb871496ae [LEADER]: Connected to new peer: Peer: permanent_uuid: "131b1371939e4d14ac8b8dfa9638fbf6" member_type: VOTER last_known_addr { host: "127.20.186.188" port: 40445 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
01:37:59.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.274771 21469 sys_catalog.cc:455] T 00000000000000000000000000000000 P 131b1371939e4d14ac8b8dfa9638fbf6 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 720cceaedd1448b39b341feb871496ae. Latest consensus state: current_term: 1 leader_uuid: "720cceaedd1448b39b341feb871496ae" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "720cceaedd1448b39b341feb871496ae" member_type: VOTER last_known_addr { host: "127.20.186.190" port: 35505 } } peers { permanent_uuid: "8c0053845799434e955627d7f5ad46de" member_type: VOTER last_known_addr { host: "127.20.186.189" port: 38631 } } peers { permanent_uuid: "131b1371939e4d14ac8b8dfa9638fbf6" member_type: VOTER last_known_addr { host: "127.20.186.188" port: 40445 } } }
01:37:59.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.275513 21469 sys_catalog.cc:458] T 00000000000000000000000000000000 P 131b1371939e4d14ac8b8dfa9638fbf6 [sys.catalog]: This master's current role is: FOLLOWER
01:37:59.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.276278 21484 sys_catalog.cc:455] T 00000000000000000000000000000000 P 720cceaedd1448b39b341feb871496ae [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "720cceaedd1448b39b341feb871496ae" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "720cceaedd1448b39b341feb871496ae" member_type: VOTER last_known_addr { host: "127.20.186.190" port: 35505 } } peers { permanent_uuid: "8c0053845799434e955627d7f5ad46de" member_type: VOTER last_known_addr { host: "127.20.186.189" port: 38631 } } peers { permanent_uuid: "131b1371939e4d14ac8b8dfa9638fbf6" member_type: VOTER last_known_addr { host: "127.20.186.188" port: 40445 } } }
01:37:59.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.277021 21484 sys_catalog.cc:458] T 00000000000000000000000000000000 P 720cceaedd1448b39b341feb871496ae [sys.catalog]: This master's current role is: LEADER
01:37:59.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.278116 21500 mvcc.cc:205] Tried to move back new op lower bound from 7333777732485853184 to 7333777732073017344. Current Snapshot: MvccSnapshot[applied={T|T < 7333777732485853184}]
01:37:59.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.282132 21493 catalog_manager.cc:1347] Generated new cluster ID: b3b00486913f4c3cb385364bf7d550de
01:37:59.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.283185 21493 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
01:37:59.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.307036 21469 sys_catalog.cc:455] T 00000000000000000000000000000000 P 131b1371939e4d14ac8b8dfa9638fbf6 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "720cceaedd1448b39b341feb871496ae" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "720cceaedd1448b39b341feb871496ae" member_type: VOTER last_known_addr { host: "127.20.186.190" port: 35505 } } peers { permanent_uuid: "8c0053845799434e955627d7f5ad46de" member_type: VOTER last_known_addr { host: "127.20.186.189" port: 38631 } } peers { permanent_uuid: "131b1371939e4d14ac8b8dfa9638fbf6" member_type: VOTER last_known_addr { host: "127.20.186.188" port: 40445 } } }
01:37:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.309967 21469 sys_catalog.cc:458] T 00000000000000000000000000000000 P 131b1371939e4d14ac8b8dfa9638fbf6 [sys.catalog]: This master's current role is: FOLLOWER
01:37:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.338963 21493 catalog_manager.cc:1370] Generated new certificate authority record
01:37:59.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.349310 21493 catalog_manager.cc:1504] Loading token signing keys...
01:37:59.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.392917 21493 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 720cceaedd1448b39b341feb871496ae: Generated new TSK 0
01:37:59.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.403631 21493 catalog_manager.cc:1514] Initializing in-progress tserver states...
01:37:59.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.439502 21226 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu
01:37:59.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu
01:37:59.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-0/wal
01:37:59.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-0/data
01:37:59.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:37:59.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:37:59.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:37:59.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:37:59.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:37:59.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:37:59.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-0/logs
01:37:59.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-0/data/info.pb
01:37:59.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:37:59.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:37:59.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:37:59.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:37:59.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:37:59.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:37:59.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:37:59.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:37:59.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
01:37:59.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:37:59.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.186.129:0
01:37:59.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.186.129
01:37:59.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.186.129
01:37:59.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:37:59.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.186.190:35505,127.20.186.189:38631,127.20.186.188:40445
01:37:59.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.186.148:33549
01:37:59.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:37:59.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:37:59.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
01:37:59.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.441980 21381 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.186.188" port: 40445 } attempt: 7
01:37:59.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.498442 21381 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8c0053845799434e955627d7f5ad46de: Bootstrap starting.
01:37:59.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.515275 21381 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 8c0053845799434e955627d7f5ad46de: Neither blocks nor log segments found. Creating new log.
01:37:59.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.520447 21381 log.cc:824] T 00000000000000000000000000000000 P 8c0053845799434e955627d7f5ad46de: Log is configured to *not* fsync() on all Append() calls
01:37:59.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.540767 21381 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8c0053845799434e955627d7f5ad46de: No bootstrap required, opened a new log
01:37:59.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.618607 21381 raft_consensus.cc:348] T 00000000000000000000000000000000 P 8c0053845799434e955627d7f5ad46de [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "720cceaedd1448b39b341feb871496ae" member_type: VOTER last_known_addr { host: "127.20.186.190" port: 35505 } } peers { permanent_uuid: "8c0053845799434e955627d7f5ad46de" member_type: VOTER last_known_addr { host: "127.20.186.189" port: 38631 } } peers { permanent_uuid: "131b1371939e4d14ac8b8dfa9638fbf6" member_type: VOTER last_known_addr { host: "127.20.186.188" port: 40445 } }
01:37:59.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.620913 21381 raft_consensus.cc:374] T 00000000000000000000000000000000 P 8c0053845799434e955627d7f5ad46de [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:37:59.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.621840 21381 raft_consensus.cc:729] T 00000000000000000000000000000000 P 8c0053845799434e955627d7f5ad46de [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8c0053845799434e955627d7f5ad46de, State: Initialized, Role: FOLLOWER
01:37:59.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.624255 21381 consensus_queue.cc:260] T 00000000000000000000000000000000 P 8c0053845799434e955627d7f5ad46de [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "720cceaedd1448b39b341feb871496ae" member_type: VOTER last_known_addr { host: "127.20.186.190" port: 35505 } } peers { permanent_uuid: "8c0053845799434e955627d7f5ad46de" member_type: VOTER last_known_addr { host: "127.20.186.189" port: 38631 } } peers { permanent_uuid: "131b1371939e4d14ac8b8dfa9638fbf6" member_type: VOTER last_known_addr { host: "127.20.186.188" port: 40445 } }
01:37:59.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.645833 21355 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "720cceaedd1448b39b341feb871496ae" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8c0053845799434e955627d7f5ad46de"
01:37:59.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.649471 21356 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "720cceaedd1448b39b341feb871496ae" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8c0053845799434e955627d7f5ad46de" is_pre_election: true
01:37:59.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.651662 21354 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 8c0053845799434e955627d7f5ad46de [term 0 FOLLOWER]: Advancing to term 1
01:37:59.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.659121 21354 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 8c0053845799434e955627d7f5ad46de [term 1 FOLLOWER]: Refusing update from remote peer 720cceaedd1448b39b341feb871496ae: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
01:37:59.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.663374 21381 sys_catalog.cc:564] T 00000000000000000000000000000000 P 8c0053845799434e955627d7f5ad46de [sys.catalog]: configured and running, proceeding with master startup.
01:37:59.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.663858 21509 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8c0053845799434e955627d7f5ad46de [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "720cceaedd1448b39b341feb871496ae" member_type: VOTER last_known_addr { host: "127.20.186.190" port: 35505 } } peers { permanent_uuid: "8c0053845799434e955627d7f5ad46de" member_type: VOTER last_known_addr { host: "127.20.186.189" port: 38631 } } peers { permanent_uuid: "131b1371939e4d14ac8b8dfa9638fbf6" member_type: VOTER last_known_addr { host: "127.20.186.188" port: 40445 } } }
01:37:59.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.667659 21509 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8c0053845799434e955627d7f5ad46de [sys.catalog]: This master's current role is: FOLLOWER
01:37:59.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.671571 21486 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 720cceaedd1448b39b341feb871496ae [LEADER]: Connected to new peer: Peer: permanent_uuid: "8c0053845799434e955627d7f5ad46de" member_type: VOTER last_known_addr { host: "127.20.186.189" port: 38631 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
01:37:59.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.743144 21509 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8c0053845799434e955627d7f5ad46de [sys.catalog]: SysCatalogTable state changed. Reason: New leader 720cceaedd1448b39b341feb871496ae. Latest consensus state: current_term: 1 leader_uuid: "720cceaedd1448b39b341feb871496ae" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "720cceaedd1448b39b341feb871496ae" member_type: VOTER last_known_addr { host: "127.20.186.190" port: 35505 } } peers { permanent_uuid: "8c0053845799434e955627d7f5ad46de" member_type: VOTER last_known_addr { host: "127.20.186.189" port: 38631 } } peers { permanent_uuid: "131b1371939e4d14ac8b8dfa9638fbf6" member_type: VOTER last_known_addr { host: "127.20.186.188" port: 40445 } } }
01:37:59.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.743870 21509 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8c0053845799434e955627d7f5ad46de [sys.catalog]: This master's current role is: FOLLOWER
01:37:59.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.745148 21515 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8c0053845799434e955627d7f5ad46de [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "720cceaedd1448b39b341feb871496ae" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "720cceaedd1448b39b341feb871496ae" member_type: VOTER last_known_addr { host: "127.20.186.190" port: 35505 } } peers { permanent_uuid: "8c0053845799434e955627d7f5ad46de" member_type: VOTER last_known_addr { host: "127.20.186.189" port: 38631 } } peers { permanent_uuid: "131b1371939e4d14ac8b8dfa9638fbf6" member_type: VOTER last_known_addr { host: "127.20.186.188" port: 40445 } } }
01:37:59.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.746031 21515 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8c0053845799434e955627d7f5ad46de [sys.catalog]: This master's current role is: FOLLOWER
01:37:59.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:37:59.784971 21525 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
01:37:59.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:59.920416 21507 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:37:59.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:59.921308 21507 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:37:59.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:59.922176 21507 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:37:59.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:59.976480 21507 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:37:59.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:37:59.978047 21507 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.186.129
01:38:00.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:00.019841 21481 catalog_manager.cc:1259] Loaded cluster ID: b3b00486913f4c3cb385364bf7d550de
01:38:00.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:00.020536 21481 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 131b1371939e4d14ac8b8dfa9638fbf6: loading cluster ID for follower catalog manager: success
01:38:00.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:00.028867 21481 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 131b1371939e4d14ac8b8dfa9638fbf6: acquiring CA information for follower catalog manager: success
01:38:00.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:00.035331 21481 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 131b1371939e4d14ac8b8dfa9638fbf6: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
01:38:00.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:00.038326 21507 tablet_server_runner.cc:78] Tablet server non-default flags:
01:38:00.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:38:00.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.186.148:33549
01:38:00.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:38:00.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:38:00.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-0/data
01:38:00.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-0/wal
01:38:00.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:38:00.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:38:00.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.186.129:0
01:38:00.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:38:00.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:38:00.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:38:00.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-0/data/info.pb
01:38:00.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.186.129
01:38:00.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:38:00.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.186.190:35505,127.20.186.189:38631,127.20.186.188:40445
01:38:00.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:38:00.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:38:00.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:38:00.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:38:00.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:38:00.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.186.129
01:38:00.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-0/logs
01:38:00.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:38:00.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:38:00.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:38:00.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
01:38:00.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:38:00.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:38:00.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:38:00.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 00:02:55 UTC on ccycloud.quasar-qgnmee.root.comops.site
01:38:00.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675483
01:38:00.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:38:00.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:00.039741 21507 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:38:00.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:00.041766 21507 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:38:00.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:00.068024 21531 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:38:00.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:00.174815 21486 sys_catalog.cc:455] T 00000000000000000000000000000000 P 720cceaedd1448b39b341feb871496ae [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "720cceaedd1448b39b341feb871496ae" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "720cceaedd1448b39b341feb871496ae" member_type: VOTER last_known_addr { host: "127.20.186.190" port: 35505 } } peers { permanent_uuid: "8c0053845799434e955627d7f5ad46de" member_type: VOTER last_known_addr { host: "127.20.186.189" port: 38631 } } peers { permanent_uuid: "131b1371939e4d14ac8b8dfa9638fbf6" member_type: VOTER last_known_addr { host: "127.20.186.188" port: 40445 } } }
01:38:00.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:00.176668 21486 sys_catalog.cc:458] T 00000000000000000000000000000000 P 720cceaedd1448b39b341feb871496ae [sys.catalog]: This master's current role is: LEADER
01:38:00.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:00.808084 21525 catalog_manager.cc:1259] Loaded cluster ID: b3b00486913f4c3cb385364bf7d550de
01:38:00.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:00.808570 21525 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 8c0053845799434e955627d7f5ad46de: loading cluster ID for follower catalog manager: success
01:38:00.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:00.840757 21525 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 8c0053845799434e955627d7f5ad46de: acquiring CA information for follower catalog manager: success
01:38:00.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:00.851433 21525 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 8c0053845799434e955627d7f5ad46de: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
01:38:01.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:00.068871 21532 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:38:01.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:01.458909 21530 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 21507
01:38:01.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:01.781920 21507 thread.cc:636] GCE (cloud detector) Time spent creating pthread: real 1.717s user 0.539s sys 1.027s
01:38:01.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:01.782891 21507 thread.cc:602] GCE (cloud detector) Time spent starting thread: real 1.718s user 0.539s sys 1.027s
01:38:01.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:01.792941 21507 server_base.cc:1034] running on GCE node
01:38:01.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:01.796420 21541 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:38:01.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:01.799129 21507 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:38:01.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:01.813691 21507 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:38:01.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:01.815169 21507 hybrid_clock.cc:648] HybridClock initialized: now 1790473081815131 us; error 42 us; skew 500 ppm
01:38:01.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:01.816164 21507 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:38:01.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:01.835702 21507 webserver.cc:460] Webserver started at http://127.20.186.129:42799/ using document root <none> and password file <none>
01:38:01.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:01.836884 21507 fs_manager.cc:359] Metadata directory not provided
01:38:01.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:01.837138 21507 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:38:01.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:01.837620 21507 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:38:01.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:01.844355 21507 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-0/data/instance:
01:38:01.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "10e997fa15e74d30a20a9efe986fda5f"
01:38:01.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 01:38:01 on dist-test-slave-37dq"
01:38:01.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:01.845789 21507 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-0/wal/instance:
01:38:01.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "10e997fa15e74d30a20a9efe986fda5f"
01:38:01.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 01:38:01 on dist-test-slave-37dq"
01:38:01.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:01.862849 21507 fs_manager.cc:691] Time spent creating directory manager: real 0.016s user 0.007s sys 0.007s
01:38:01.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:01.894454 21546 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:38:01.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:01.897861 21507 fs_manager.cc:725] Time spent opening block manager: real 0.016s user 0.002s sys 0.002s
01:38:01.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:01.898223 21507 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-0/data,/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-0/wal
01:38:01.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "10e997fa15e74d30a20a9efe986fda5f"
01:38:01.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 01:38:01 on dist-test-slave-37dq"
01:38:01.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:01.898659 21507 fs_report.cc:383] FS layout report
01:38:01.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:38:01.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-0/wal
01:38:01.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-0/wal
01:38:01.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-0/data/data
01:38:01.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:38:01.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:38:01.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:38:01.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:38:01.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:38:01.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:38:01.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:38:01.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:38:01.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:38:01.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:38:01.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:38:02.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:02.008240 21507 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:38:02.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:02.010119 21507 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:38:02.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:02.010627 21507 kserver.cc:163] Server-wide thread pool size limit: 3276
01:38:02.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:02.021618 21507 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
01:38:02.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:02.033689 21507 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
01:38:02.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:02.033946 21507 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
01:38:02.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:02.034358 21507 ts_tablet_manager.cc:610] Registered 0 tablets
01:38:02.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:02.034560 21507 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
01:38:02.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:02.345142 21507 rpc_server.cc:307] RPC server started. Bound to: 127.20.186.129:43949
01:38:02.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:02.345314 21658 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.186.129:43949 every 8 connection(s)
01:38:02.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:02.347669 21507 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-0/data/info.pb
01:38:02.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:02.349320 21226 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu as pid 21507
01:38:02.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:02.349870 21226 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-0/wal/instance
01:38:02.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:02.365236 21226 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu
01:38:02.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu
01:38:02.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-1/wal
01:38:02.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-1/data
01:38:02.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:38:02.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:38:02.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:38:02.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:38:02.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:38:02.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:38:02.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-1/logs
01:38:02.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-1/data/info.pb
01:38:02.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:38:02.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:38:02.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:38:02.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:38:02.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:38:02.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:38:02.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:38:02.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:38:02.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
01:38:02.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:38:02.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.186.130:0
01:38:02.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.186.130
01:38:02.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.186.130
01:38:02.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:38:02.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.186.190:35505,127.20.186.189:38631,127.20.186.188:40445
01:38:02.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.186.148:33549
01:38:02.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:38:02.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:38:02.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
01:38:02.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:02.409734 21659 heartbeater.cc:347] Connected to a master server at 127.20.186.188:40445
01:38:02.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:02.410161 21659 heartbeater.cc:464] Registering TS with master...
01:38:02.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:02.411316 21659 heartbeater.cc:511] Master 127.20.186.188:40445 requested a full tablet report, sending...
01:38:02.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:02.415364 21425 ts_manager.cc:194] Registered new tserver with Master: 10e997fa15e74d30a20a9efe986fda5f (127.20.186.129:43949)
01:38:02.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:02.438347 21664 heartbeater.cc:347] Connected to a master server at 127.20.186.190:35505
01:38:02.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:02.438689 21664 heartbeater.cc:464] Registering TS with master...
01:38:02.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:02.439522 21664 heartbeater.cc:511] Master 127.20.186.190:35505 requested a full tablet report, sending...
01:38:02.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:02.441849 21271 ts_manager.cc:194] Registered new tserver with Master: 10e997fa15e74d30a20a9efe986fda5f (127.20.186.129:43949)
01:38:02.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:02.444509 21660 heartbeater.cc:347] Connected to a master server at 127.20.186.189:38631
01:38:02.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:02.444855 21660 heartbeater.cc:464] Registering TS with master...
01:38:02.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:02.444731 21271 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.20.186.129:43679
01:38:02.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:02.445593 21660 heartbeater.cc:511] Master 127.20.186.189:38631 requested a full tablet report, sending...
01:38:02.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:02.448700 21346 ts_manager.cc:194] Registered new tserver with Master: 10e997fa15e74d30a20a9efe986fda5f (127.20.186.129:43949)
01:38:02.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:02.725133 21663 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:38:02.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:02.725698 21663 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:38:02.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:02.726259 21663 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:38:02.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:02.757354 21663 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:38:02.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:02.758261 21663 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.186.130
01:38:02.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:02.793854 21663 tablet_server_runner.cc:78] Tablet server non-default flags:
01:38:02.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:38:02.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.186.148:33549
01:38:02.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:38:02.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:38:02.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-1/data
01:38:02.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-1/wal
01:38:02.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:38:02.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:38:02.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.186.130:0
01:38:02.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:38:02.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:38:02.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:38:02.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-1/data/info.pb
01:38:02.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.186.130
01:38:02.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:38:02.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.186.190:35505,127.20.186.189:38631,127.20.186.188:40445
01:38:02.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:38:02.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:38:02.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:38:02.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:38:02.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:38:02.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.186.130
01:38:02.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-1/logs
01:38:02.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:38:02.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:38:02.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:38:02.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
01:38:02.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:38:02.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:38:02.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:38:02.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 00:02:55 UTC on ccycloud.quasar-qgnmee.root.comops.site
01:38:02.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675483
01:38:02.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:38:02.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:02.795318 21663 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:38:02.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:02.797078 21663 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:38:02.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:02.818001 21676 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:38:03.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:03.449081 21664 heartbeater.cc:503] Master 127.20.186.190:35505 was elected leader, sending a full tablet report...
01:38:04.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:02.818370 21675 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:38:04.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:02.819712 21678 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:38:04.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:04.035876 21677 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
01:38:04.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.035988 21663 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:38:04.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.041744 21663 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:38:04.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.044255 21663 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:38:04.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.045815 21663 hybrid_clock.cc:648] HybridClock initialized: now 1790473084045766 us; error 46 us; skew 500 ppm
01:38:04.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.046617 21663 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:38:04.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.056592 21663 webserver.cc:460] Webserver started at http://127.20.186.130:38989/ using document root <none> and password file <none>
01:38:04.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.057489 21663 fs_manager.cc:359] Metadata directory not provided
01:38:04.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.057721 21663 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:38:04.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.058184 21663 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:38:04.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.062866 21663 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-1/data/instance:
01:38:04.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2752300a83af4d0ab519b7e19a92b74c"
01:38:04.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 01:38:04 on dist-test-slave-37dq"
01:38:04.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.063930 21663 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-1/wal/instance:
01:38:04.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2752300a83af4d0ab519b7e19a92b74c"
01:38:04.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 01:38:04 on dist-test-slave-37dq"
01:38:04.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.071118 21663 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
01:38:04.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.076402 21687 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:38:04.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.077381 21663 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
01:38:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.077702 21663 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-1/data,/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-1/wal
01:38:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2752300a83af4d0ab519b7e19a92b74c"
01:38:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 01:38:04 on dist-test-slave-37dq"
01:38:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.078050 21663 fs_report.cc:383] FS layout report
01:38:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:38:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-1/wal
01:38:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-1/wal
01:38:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-1/data/data
01:38:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:38:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:38:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:38:04.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:38:04.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:38:04.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:38:04.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:38:04.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:38:04.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:38:04.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:38:04.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:38:04.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.116771 21663 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:38:04.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.118183 21663 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:38:04.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.118643 21663 kserver.cc:163] Server-wide thread pool size limit: 3276
01:38:04.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.121227 21663 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
01:38:04.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.125418 21663 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
01:38:04.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.125646 21663 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
01:38:04.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.125954 21663 ts_tablet_manager.cc:610] Registered 0 tablets
01:38:04.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.126139 21663 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
01:38:04.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.267899 21663 rpc_server.cc:307] RPC server started. Bound to: 127.20.186.130:32843
01:38:04.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.268045 21799 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.186.130:32843 every 8 connection(s)
01:38:04.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.270411 21663 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-1/data/info.pb
01:38:04.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.271196 21226 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu as pid 21663
01:38:04.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.271687 21226 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-1/wal/instance
01:38:04.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.292695 21226 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu
01:38:04.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu
01:38:04.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-2/wal
01:38:04.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-2/data
01:38:04.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:38:04.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:38:04.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:38:04.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:38:04.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:38:04.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:38:04.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-2/logs
01:38:04.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-2/data/info.pb
01:38:04.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:38:04.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:38:04.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:38:04.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:38:04.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:38:04.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:38:04.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:38:04.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:38:04.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
01:38:04.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:38:04.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.186.131:0
01:38:04.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.186.131
01:38:04.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.186.131
01:38:04.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:38:04.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.186.190:35505,127.20.186.189:38631,127.20.186.188:40445
01:38:04.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.186.148:33549
01:38:04.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:38:04.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:38:04.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
01:38:04.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.327309 21801 heartbeater.cc:347] Connected to a master server at 127.20.186.189:38631
01:38:04.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.327736 21801 heartbeater.cc:464] Registering TS with master...
01:38:04.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.328600 21800 heartbeater.cc:347] Connected to a master server at 127.20.186.188:40445
01:38:04.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.328938 21800 heartbeater.cc:464] Registering TS with master...
01:38:04.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.328890 21801 heartbeater.cc:511] Master 127.20.186.189:38631 requested a full tablet report, sending...
01:38:04.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.329742 21800 heartbeater.cc:511] Master 127.20.186.188:40445 requested a full tablet report, sending...
01:38:04.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.331912 21346 ts_manager.cc:194] Registered new tserver with Master: 2752300a83af4d0ab519b7e19a92b74c (127.20.186.130:32843)
01:38:04.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.332267 21425 ts_manager.cc:194] Registered new tserver with Master: 2752300a83af4d0ab519b7e19a92b74c (127.20.186.130:32843)
01:38:04.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.339010 21802 heartbeater.cc:347] Connected to a master server at 127.20.186.190:35505
01:38:04.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.339344 21802 heartbeater.cc:464] Registering TS with master...
01:38:04.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.340093 21802 heartbeater.cc:511] Master 127.20.186.190:35505 requested a full tablet report, sending...
01:38:04.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.342453 21271 ts_manager.cc:194] Registered new tserver with Master: 2752300a83af4d0ab519b7e19a92b74c (127.20.186.130:32843)
01:38:04.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.344712 21271 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.20.186.130:59951
01:38:04.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:04.622808 21807 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:38:04.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:04.623260 21807 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:38:04.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:04.623685 21807 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:38:04.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:04.653919 21807 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:38:04.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:04.654682 21807 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.186.131
01:38:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.687845 21807 tablet_server_runner.cc:78] Tablet server non-default flags:
01:38:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:38:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.186.148:33549
01:38:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:38:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:38:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-2/data
01:38:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-2/wal
01:38:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:38:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:38:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.186.131:0
01:38:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:38:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:38:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:38:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-2/data/info.pb
01:38:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.186.131
01:38:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:38:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.186.190:35505,127.20.186.189:38631,127.20.186.188:40445
01:38:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:38:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:38:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:38:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:38:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:38:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.186.131
01:38:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-2/logs
01:38:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:38:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:38:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:38:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
01:38:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:38:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:38:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:38:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 00:02:55 UTC on ccycloud.quasar-qgnmee.root.comops.site
01:38:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675483
01:38:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:38:04.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.689019 21807 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:38:04.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.690461 21807 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:38:04.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:04.710273 21817 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:38:05.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:05.348873 21802 heartbeater.cc:503] Master 127.20.186.190:35505 was elected leader, sending a full tablet report...
01:38:05.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:04.710318 21819 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:38:05.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:04.711927 21807 server_base.cc:1034] running on GCE node
01:38:05.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:04.710343 21816 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:38:05.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:05.900969 21807 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:38:05.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:05.903235 21807 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:38:05.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:05.904636 21807 hybrid_clock.cc:648] HybridClock initialized: now 1790473085904600 us; error 44 us; skew 500 ppm
01:38:05.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:05.905566 21807 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:38:05.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:05.911792 21807 webserver.cc:460] Webserver started at http://127.20.186.131:42079/ using document root <none> and password file <none>
01:38:05.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:05.912760 21807 fs_manager.cc:359] Metadata directory not provided
01:38:05.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:05.913051 21807 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:38:05.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:05.913503 21807 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:38:05.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:05.918155 21807 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-2/data/instance:
01:38:05.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a463f44101d146cfa622f05cf8c6cc70"
01:38:05.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 01:38:05 on dist-test-slave-37dq"
01:38:05.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:05.919199 21807 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-2/wal/instance:
01:38:05.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a463f44101d146cfa622f05cf8c6cc70"
01:38:05.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 01:38:05 on dist-test-slave-37dq"
01:38:05.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:05.926180 21807 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.006s sys 0.003s
01:38:05.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:05.931330 21827 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:38:05.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:05.932339 21807 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
01:38:05.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:05.932682 21807 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-2/data,/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-2/wal
01:38:05.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a463f44101d146cfa622f05cf8c6cc70"
01:38:05.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 01:38:05 on dist-test-slave-37dq"
01:38:05.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:05.933027 21807 fs_report.cc:383] FS layout report
01:38:05.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:38:05.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-2/wal
01:38:05.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-2/wal
01:38:05.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-2/data/data
01:38:05.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:38:05.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:38:05.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:38:05.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:38:05.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:38:05.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:38:05.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:38:05.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:38:05.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:38:05.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:38:05.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:38:05.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:05.967638 21807 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:38:05.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:05.969136 21807 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:38:05.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:05.969610 21807 kserver.cc:163] Server-wide thread pool size limit: 3276
01:38:05.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:05.972394 21807 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
01:38:05.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:05.976348 21807 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
01:38:05.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:05.976578 21807 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.001s sys 0.000s
01:38:05.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:05.976879 21807 ts_tablet_manager.cc:610] Registered 0 tablets
01:38:05.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:05.977052 21807 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
01:38:06.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:06.127604 21807 rpc_server.cc:307] RPC server started. Bound to: 127.20.186.131:44571
01:38:06.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:06.127754 21939 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.186.131:44571 every 8 connection(s)
01:38:06.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:06.130120 21807 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-2/data/info.pb
01:38:06.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:06.132907 21226 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu as pid 21807
01:38:06.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:06.133426 21226 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster5536789500732785753/ts-2/wal/instance
01:38:06.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:06.179136 21940 heartbeater.cc:347] Connected to a master server at 127.20.186.188:40445
01:38:06.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:06.179554 21940 heartbeater.cc:464] Registering TS with master...
01:38:06.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:06.180119 21941 heartbeater.cc:347] Connected to a master server at 127.20.186.189:38631
01:38:06.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:06.180485 21941 heartbeater.cc:464] Registering TS with master...
01:38:06.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:06.180852 21940 heartbeater.cc:511] Master 127.20.186.188:40445 requested a full tablet report, sending...
01:38:06.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:06.181334 21941 heartbeater.cc:511] Master 127.20.186.189:38631 requested a full tablet report, sending...
01:38:06.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:06.183907 21425 ts_manager.cc:194] Registered new tserver with Master: a463f44101d146cfa622f05cf8c6cc70 (127.20.186.131:44571)
01:38:06.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:06.184122 21942 heartbeater.cc:347] Connected to a master server at 127.20.186.190:35505
01:38:06.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:06.184252 21346 ts_manager.cc:194] Registered new tserver with Master: a463f44101d146cfa622f05cf8c6cc70 (127.20.186.131:44571)
01:38:06.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:06.184500 21942 heartbeater.cc:464] Registering TS with master...
01:38:06.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:06.185328 21942 heartbeater.cc:511] Master 127.20.186.190:35505 requested a full tablet report, sending...
01:38:06.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:06.187110 21271 ts_manager.cc:194] Registered new tserver with Master: a463f44101d146cfa622f05cf8c6cc70 (127.20.186.131:44571)
01:38:06.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:06.188514 21271 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.20.186.131:33427
01:38:06.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:06.202661 21226 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
01:38:06.213 [DEBUG - main] (MiniKuduCluster.java:195) Response:
01:38:06.213 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
01:38:06.214 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.20.186.190"
port: 35505
}
bound_http_address {
host: "127.20.186.190"
port: 36919
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.20.186.189"
port: 38631
}
bound_http_address {
host: "127.20.186.189"
port: 33429
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.20.186.188"
port: 40445
}
bound_http_address {
host: "127.20.186.188"
port: 44867
}
}
}
01:38:06.214 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
01:38:06.215 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.20.186.129"
port: 43949
}
bound_http_address {
host: "127.20.186.129"
port: 42799
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.20.186.130"
port: 32843
}
bound_http_address {
host: "127.20.186.130"
port: 38989
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.20.186.131"
port: 44571
}
bound_http_address {
host: "127.20.186.131"
port: 42079
}
}
}
01:38:06.215 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
01:38:06.216 [DEBUG - main] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster4891687883482218832/krb5kdc/krb5cc
01:38:06.216 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
01:38:06.217 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskoFx69z/test-tmp
01:38:06.217 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
01:38:06.217 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskoFx69z/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
01:38:06.219 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
01:38:06.536 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:38:06.537 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
01:38:06.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:06.539637 21951 mini_oidc.cc:152] Starting JWKS server
01:38:06.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:06.543704 21951 webserver.cc:460] Webserver started at http://127.0.0.1:46859/ using document root <none> and password file <none>
01:38:06.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:06.544262 21951 mini_oidc.cc:189] Starting OIDC Discovery server
01:38:06.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:06.546561 21951 webserver.cc:460] Webserver started at http://127.0.0.1:40979/ using document root <none> and password file <none>
01:38:06.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
01:38:06.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/krb5kdc/principal' for realm 'KRBTEST.COM',
01:38:06.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
01:38:06.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 01:38:06 dist-test-slave-37dq krb5kdc[21962](info): setting up network...
01:38:06.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
01:38:06.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 01:38:06 dist-test-slave-37dq krb5kdc[21962](info): set up 2 sockets
01:38:06.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 01:38:06 dist-test-slave-37dq krb5kdc[21962](info): commencing operation
01:38:06.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
01:38:07.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:07.191799 21942 heartbeater.cc:503] Master 127.20.186.190:35505 was elected leader, sending a full tablet report...
01:38:07.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:07.660960 21951 mini_kdc.cc:121] Time spent starting KDC: real 1.090s user 0.005s sys 0.013s
01:38:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
01:38:07.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
01:38:07.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
01:38:07.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
01:38:07.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
01:38:07.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
01:38:07.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
01:38:07.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
01:38:07.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
01:38:07.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
01:38:07.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/krb5kdc/test-user.keytab.
01:38:07.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/krb5kdc/test-user.keytab.
01:38:07.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 01:38:07 dist-test-slave-37dq krb5kdc[21962](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790473087, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:38:07.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
01:38:07.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T01:38:07Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
01:38:07.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T01:38:07Z Disabled control of system clock
01:38:07.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.21.111.254@KRBTEST.COM; defaulting to no policy
01:38:07.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:38:07.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.21.111.254@KRBTEST.COM" created.
01:38:07.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:38:07.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.21.111.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/krb5kdc/kudu_127.21.111.254.keytab.
01:38:07.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.21.111.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/krb5kdc/kudu_127.21.111.254.keytab.
01:38:07.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:07.845475 21951 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu
01:38:07.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu
01:38:07.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-0/wal
01:38:07.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-0/data
01:38:07.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:38:07.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:38:07.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:38:07.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:38:07.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:38:07.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:38:07.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-0/logs
01:38:07.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-0/data/info.pb
01:38:07.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:38:07.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:38:07.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:38:07.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:38:07.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:38:07.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:38:07.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:38:07.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:38:07.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:38:07.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:38:07.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:38:07.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:38:07.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.111.254:42895
01:38:07.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.111.254
01:38:07.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:38:07.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/krb5kdc/kudu_127.21.111.254.keytab
01:38:07.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.21.111.254
01:38:07.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:38:07.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:38:07.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:38:07.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.111.212:37441
01:38:07.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:38:07.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:38:07.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:38:07.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:38:07.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.111.254:42895,127.21.111.253:37139,127.21.111.252:39629
01:38:07.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:38:07.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46859/jwks.json
01:38:07.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:38:08.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:08.163290 21976 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:38:08.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:08.163929 21976 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:38:08.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:08.164741 21976 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:38:08.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:08.198597 21976 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:38:08.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:08.198889 21976 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:38:08.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:08.199369 21976 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:38:08.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:08.199717 21976 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:38:08.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:08.200014 21976 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:38:08.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:08.200307 21976 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:38:08.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:08.200526 21976 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46859/jwks.json
01:38:08.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:08.237841 21976 master_runner.cc:386] Master server non-default flags:
01:38:08.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:38:08.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.111.212:37441
01:38:08.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:38:08.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:38:08.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-0/data
01:38:08.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-0/wal
01:38:08.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:38:08.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:38:08.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.111.254:42895,127.21.111.253:37139,127.21.111.252:39629
01:38:08.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/krb5kdc/kudu_127.21.111.254.keytab
01:38:08.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.21.111.254
01:38:08.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:38:08.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:38:08.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:38:08.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.111.254:42895
01:38:08.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:38:08.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:38:08.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:38:08.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46859/jwks.json
01:38:08.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:38:08.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:38:08.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:38:08.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:38:08.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:38:08.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-0/data/info.pb
01:38:08.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.111.254
01:38:08.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:38:08.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:38:08.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:38:08.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:38:08.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:38:08.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:38:08.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-0/logs
01:38:08.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:38:08.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:38:08.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:38:08.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:38:08.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:38:08.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:38:08.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:38:08.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 00:02:55 UTC on ccycloud.quasar-qgnmee.root.comops.site
01:38:08.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675483
01:38:08.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:38:08.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:08.239291 21976 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:38:08.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:08.240386 21976 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:38:08.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:08.257254 21983 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:38:09.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:08.257372 21982 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:38:09.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:08.257994 21985 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:38:09.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:09.515374 21984 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1255 milliseconds
01:38:09.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:09.515487 21976 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:38:09.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:09.516958 21976 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:38:09.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:09.519667 21976 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:38:09.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:09.521018 21976 hybrid_clock.cc:648] HybridClock initialized: now 1790473089520989 us; error 29 us; skew 500 ppm
01:38:09.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:09.521909 21976 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:38:09.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 01:38:09 dist-test-slave-37dq krb5kdc[21962](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790473089, etypes {rep=17 tkt=17 ses=17}, kudu/127.21.111.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:38:09.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:09.531199 21976 init.cc:377] Logged in from keytab as kudu/127.21.111.254@KRBTEST.COM (short username kudu)
01:38:09.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:09.536705 21976 webserver.cc:460] Webserver started at http://127.21.111.254:45347/ using document root <none> and password file <none>
01:38:09.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:09.537580 21976 fs_manager.cc:359] Metadata directory not provided
01:38:09.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:09.537803 21976 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:38:09.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:09.538254 21976 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:38:09.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:09.542945 21976 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-0/data/instance:
01:38:09.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9a0e4466388e452592121c40b5e26931"
01:38:09.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 01:38:09 on dist-test-slave-37dq"
01:38:09.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:09.544008 21976 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-0/wal/instance:
01:38:09.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9a0e4466388e452592121c40b5e26931"
01:38:09.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 01:38:09 on dist-test-slave-37dq"
01:38:09.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:09.550817 21976 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.005s sys 0.000s
01:38:09.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:09.556677 21993 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:38:09.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:09.557683 21976 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
01:38:09.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:09.558041 21976 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-0/data,/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-0/wal
01:38:09.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9a0e4466388e452592121c40b5e26931"
01:38:09.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 01:38:09 on dist-test-slave-37dq"
01:38:09.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:09.558382 21976 fs_report.cc:383] FS layout report
01:38:09.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:38:09.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-0/wal
01:38:09.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-0/wal
01:38:09.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-0/data/data
01:38:09.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:38:09.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:38:09.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:38:09.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:38:09.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:38:09.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:38:09.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:38:09.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:38:09.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:38:09.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:38:09.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:38:09.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:09.592338 21976 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:38:09.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:09.598011 21976 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:38:09.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:09.598490 21976 kserver.cc:163] Server-wide thread pool size limit: 3276
01:38:09.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:09.670133 21976 rpc_server.cc:307] RPC server started. Bound to: 127.21.111.254:42895
01:38:09.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:09.670225 22044 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.111.254:42895 every 8 connection(s)
01:38:09.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:09.672897 21976 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-0/data/info.pb
01:38:09.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:09.677914 22045 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:38:09.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:09.678897 21951 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu as pid 21976
01:38:09.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:09.679440 21951 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-0/wal/instance
01:38:09.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:09.690596 22045 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.111.254" port: 42895 } has no permanent_uuid. Determining permanent_uuid...
01:38:09.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.21.111.253@KRBTEST.COM; defaulting to no policy
01:38:09.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:38:09.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.21.111.253@KRBTEST.COM" created.
01:38:09.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 01:38:09 dist-test-slave-37dq krb5kdc[21962](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790473089, etypes {rep=17 tkt=17 ses=17}, kudu/127.21.111.254@KRBTEST.COM for kudu/127.21.111.254@KRBTEST.COM
01:38:09.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:38:09.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.21.111.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/krb5kdc/kudu_127.21.111.253.keytab.
01:38:09.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.21.111.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/krb5kdc/kudu_127.21.111.253.keytab.
01:38:09.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:09.725941 21951 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu
01:38:09.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu
01:38:09.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-1/wal
01:38:09.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-1/data
01:38:09.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:38:09.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:38:09.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:38:09.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:38:09.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:38:09.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:38:09.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-1/logs
01:38:09.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-1/data/info.pb
01:38:09.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:38:09.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:38:09.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:38:09.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:38:09.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:38:09.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:38:09.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:38:09.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:38:09.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:38:09.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:38:09.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:38:09.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:38:09.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.111.253:37139
01:38:09.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.111.253
01:38:09.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:38:09.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/krb5kdc/kudu_127.21.111.253.keytab
01:38:09.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.21.111.253
01:38:09.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:38:09.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:38:09.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:38:09.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.111.212:37441
01:38:09.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:38:09.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:38:09.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:38:09.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:38:09.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.111.254:42895,127.21.111.253:37139,127.21.111.252:39629
01:38:09.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:38:09.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46859/jwks.json
01:38:09.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:38:09.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:09.734651 22045 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } has no permanent_uuid. Determining permanent_uuid...
01:38:09.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:09.736948 21996 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.111.253:37139: connect: Connection refused (error 111)
01:38:09.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:09.741032 22045 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.111.253:37139: Network error: Client connection negotiation failed: client connection to 127.21.111.253:37139: connect: Connection refused (error 111)
01:38:09.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:09.790541 22045 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } attempt: 1
01:38:09.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:09.794258 22045 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.111.253:37139: Network error: Client connection negotiation failed: client connection to 127.21.111.253:37139: connect: Connection refused (error 111)
01:38:09.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:09.862694 22045 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } attempt: 2
01:38:09.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:09.866518 22045 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.111.253:37139: Network error: Client connection negotiation failed: client connection to 127.21.111.253:37139: connect: Connection refused (error 111)
01:38:09.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:09.957924 22045 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } attempt: 3
01:38:09.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:09.961717 22045 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.111.253:37139: Network error: Client connection negotiation failed: client connection to 127.21.111.253:37139: connect: Connection refused (error 111)
01:38:10.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:10.037213 22051 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:38:10.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:10.037846 22051 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:38:10.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:10.038517 22051 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:38:10.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:10.071107 22051 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:38:10.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:10.071378 22051 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:38:10.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:10.071801 22051 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:38:10.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:10.072141 22051 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:38:10.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:10.072427 22051 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:38:10.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:10.072754 22051 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:38:10.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:10.073005 22051 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46859/jwks.json
01:38:10.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:10.105098 22045 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } attempt: 4
01:38:10.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:10.109298 22045 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.111.253:37139: Network error: Client connection negotiation failed: client connection to 127.21.111.253:37139: connect: Connection refused (error 111)
01:38:10.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:10.121088 22051 master_runner.cc:386] Master server non-default flags:
01:38:10.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:38:10.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.111.212:37441
01:38:10.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:38:10.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:38:10.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-1/data
01:38:10.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-1/wal
01:38:10.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:38:10.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:38:10.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.111.254:42895,127.21.111.253:37139,127.21.111.252:39629
01:38:10.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/krb5kdc/kudu_127.21.111.253.keytab
01:38:10.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.21.111.253
01:38:10.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:38:10.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:38:10.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:38:10.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.111.253:37139
01:38:10.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:38:10.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:38:10.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:38:10.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46859/jwks.json
01:38:10.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:38:10.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:38:10.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:38:10.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:38:10.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:38:10.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-1/data/info.pb
01:38:10.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.111.253
01:38:10.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:38:10.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:38:10.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:38:10.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:38:10.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:38:10.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:38:10.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-1/logs
01:38:10.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:38:10.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:38:10.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:38:10.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:38:10.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:38:10.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:38:10.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:38:10.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 00:02:55 UTC on ccycloud.quasar-qgnmee.root.comops.site
01:38:10.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675483
01:38:10.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:38:10.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:10.122557 22051 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:38:10.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:10.123668 22051 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:38:10.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:10.141671 22062 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:38:10.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:10.408711 22045 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } attempt: 5
01:38:10.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:10.413386 22045 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.111.253:37139: Network error: Client connection negotiation failed: client connection to 127.21.111.253:37139: connect: Connection refused (error 111)
01:38:10.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:10.960809 22045 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } attempt: 6
01:38:10.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:10.967104 22045 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.111.253:37139: Network error: Client connection negotiation failed: client connection to 127.21.111.253:37139: connect: Connection refused (error 111)
01:38:11.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:10.142819 22060 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:38:11.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:10.143409 22051 server_base.cc:1034] running on GCE node
01:38:11.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:10.141990 22059 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:38:11.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:11.370141 22051 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:38:11.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:11.373049 22051 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:38:11.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:11.374475 22051 hybrid_clock.cc:648] HybridClock initialized: now 1790473091374409 us; error 68 us; skew 500 ppm
01:38:11.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:11.375353 22051 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:38:11.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 01:38:11 dist-test-slave-37dq krb5kdc[21962](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790473091, etypes {rep=17 tkt=17 ses=17}, kudu/127.21.111.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:38:11.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:11.381927 22051 init.cc:377] Logged in from keytab as kudu/127.21.111.253@KRBTEST.COM (short username kudu)
01:38:11.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:11.387841 22051 webserver.cc:460] Webserver started at http://127.21.111.253:41045/ using document root <none> and password file <none>
01:38:11.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:11.388821 22051 fs_manager.cc:359] Metadata directory not provided
01:38:11.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:11.389034 22051 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:38:11.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:11.389483 22051 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:38:11.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:11.394191 22051 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-1/data/instance:
01:38:11.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0fd65b39ee41409a85c6a728d91f4051"
01:38:11.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 01:38:11 on dist-test-slave-37dq"
01:38:11.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:11.395277 22051 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-1/wal/instance:
01:38:11.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0fd65b39ee41409a85c6a728d91f4051"
01:38:11.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 01:38:11 on dist-test-slave-37dq"
01:38:11.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:11.402784 22051 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
01:38:11.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:11.408051 22073 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:38:11.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:11.409097 22051 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
01:38:11.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:11.409453 22051 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-1/data,/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-1/wal
01:38:11.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0fd65b39ee41409a85c6a728d91f4051"
01:38:11.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 01:38:11 on dist-test-slave-37dq"
01:38:11.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:11.409808 22051 fs_report.cc:383] FS layout report
01:38:11.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:38:11.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-1/wal
01:38:11.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-1/wal
01:38:11.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-1/data/data
01:38:11.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:38:11.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:38:11.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:38:11.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:38:11.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:38:11.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:38:11.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:38:11.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:38:11.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:38:11.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:38:11.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:38:11.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:11.446312 22051 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:38:11.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:11.452317 22051 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:38:11.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:11.452831 22051 kserver.cc:163] Server-wide thread pool size limit: 3276
01:38:11.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:11.523489 22051 rpc_server.cc:307] RPC server started. Bound to: 127.21.111.253:37139
01:38:11.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:11.523568 22124 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.111.253:37139 every 8 connection(s)
01:38:11.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:11.526069 22051 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-1/data/info.pb
01:38:11.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:11.531085 22125 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:38:11.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:11.533193 21951 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu as pid 22051
01:38:11.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:11.533584 21951 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-1/wal/instance
01:38:11.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:11.543571 22125 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.111.254" port: 42895 } has no permanent_uuid. Determining permanent_uuid...
01:38:11.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.21.111.252@KRBTEST.COM; defaulting to no policy
01:38:11.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:38:11.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.21.111.252@KRBTEST.COM" created.
01:38:11.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 01:38:11 dist-test-slave-37dq krb5kdc[21962](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790473091, etypes {rep=17 tkt=17 ses=17}, kudu/127.21.111.253@KRBTEST.COM for kudu/127.21.111.254@KRBTEST.COM
01:38:11.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:38:11.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.21.111.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/krb5kdc/kudu_127.21.111.252.keytab.
01:38:11.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.21.111.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/krb5kdc/kudu_127.21.111.252.keytab.
01:38:11.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:11.576248 21951 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu
01:38:11.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu
01:38:11.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-2/wal
01:38:11.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-2/data
01:38:11.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:38:11.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:38:11.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:38:11.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:38:11.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:38:11.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:38:11.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-2/logs
01:38:11.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-2/data/info.pb
01:38:11.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:38:11.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:38:11.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:38:11.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:38:11.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:38:11.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:38:11.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:38:11.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:38:11.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:38:11.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:38:11.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:38:11.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:38:11.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.111.252:39629
01:38:11.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.111.252
01:38:11.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:38:11.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/krb5kdc/kudu_127.21.111.252.keytab
01:38:11.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.21.111.252
01:38:11.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:38:11.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:38:11.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:38:11.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.111.212:37441
01:38:11.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:38:11.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:38:11.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:38:11.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:38:11.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.111.254:42895,127.21.111.253:37139,127.21.111.252:39629
01:38:11.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:38:11.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46859/jwks.json
01:38:11.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:38:11.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:11.581265 22125 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } has no permanent_uuid. Determining permanent_uuid...
01:38:11.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 01:38:11 dist-test-slave-37dq krb5kdc[21962](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790473091, etypes {rep=17 tkt=17 ses=17}, kudu/127.21.111.253@KRBTEST.COM for kudu/127.21.111.253@KRBTEST.COM
01:38:11.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:11.607980 22125 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } has no permanent_uuid. Determining permanent_uuid...
01:38:11.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:11.610105 22076 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.111.252:39629: connect: Connection refused (error 111)
01:38:11.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:11.613967 22125 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.111.252:39629: Network error: Client connection negotiation failed: client connection to 127.21.111.252:39629: connect: Connection refused (error 111)
01:38:11.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:11.663460 22125 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } attempt: 1
01:38:11.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:11.668387 22125 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.111.252:39629: Network error: Client connection negotiation failed: client connection to 127.21.111.252:39629: connect: Connection refused (error 111)
01:38:11.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:11.736866 22125 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } attempt: 2
01:38:11.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:11.740690 22125 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.111.252:39629: Network error: Client connection negotiation failed: client connection to 127.21.111.252:39629: connect: Connection refused (error 111)
01:38:11.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:11.832118 22125 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } attempt: 3
01:38:11.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:11.835650 22125 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.111.252:39629: Network error: Client connection negotiation failed: client connection to 127.21.111.252:39629: connect: Connection refused (error 111)
01:38:11.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:11.882176 22131 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:38:11.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:11.882787 22131 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:38:11.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:11.883433 22131 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:38:11.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:11.914793 22131 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:38:11.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:11.915014 22131 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:38:11.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:11.915270 22131 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:38:11.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:11.915472 22131 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:38:11.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:11.915781 22131 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:38:11.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:11.916152 22131 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:38:11.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:11.916445 22131 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46859/jwks.json
01:38:11.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:11.951988 22131 master_runner.cc:386] Master server non-default flags:
01:38:11.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:38:11.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.111.212:37441
01:38:11.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:38:11.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:38:11.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-2/data
01:38:11.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-2/wal
01:38:11.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:38:11.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:38:11.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.111.254:42895,127.21.111.253:37139,127.21.111.252:39629
01:38:11.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/krb5kdc/kudu_127.21.111.252.keytab
01:38:11.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.21.111.252
01:38:11.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:38:11.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:38:11.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:38:11.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.111.252:39629
01:38:11.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:38:11.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:38:11.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:38:11.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46859/jwks.json
01:38:11.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:38:11.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:38:11.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:38:11.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:38:11.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:38:11.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-2/data/info.pb
01:38:11.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.111.252
01:38:11.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:38:11.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:38:11.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:38:11.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:38:11.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:38:11.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:38:11.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-2/logs
01:38:11.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:38:11.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:38:11.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:38:11.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:38:11.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:38:11.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:38:11.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:38:11.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 00:02:55 UTC on ccycloud.quasar-qgnmee.root.comops.site
01:38:11.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675483
01:38:11.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:38:11.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:11.953431 22131 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:38:11.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:11.954346 22131 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:38:11.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:11.972101 22140 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:38:11.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:11.979043 22125 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } attempt: 4
01:38:11.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:11.983083 22125 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.111.252:39629: Network error: Client connection negotiation failed: client connection to 127.21.111.252:39629: connect: Connection refused (error 111)
01:38:12.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:12.027680 22045 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } attempt: 7
01:38:12.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 01:38:12 dist-test-slave-37dq krb5kdc[21962](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790473089, etypes {rep=17 tkt=17 ses=17}, kudu/127.21.111.254@KRBTEST.COM for kudu/127.21.111.253@KRBTEST.COM
01:38:12.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:12.054056 22045 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } has no permanent_uuid. Determining permanent_uuid...
01:38:12.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:12.058915 22045 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.111.252:39629: Network error: Client connection negotiation failed: client connection to 127.21.111.252:39629: connect: Connection refused (error 111)
01:38:12.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:12.117487 22045 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } attempt: 1
01:38:12.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:12.120918 22045 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.111.252:39629: Network error: Client connection negotiation failed: client connection to 127.21.111.252:39629: connect: Connection refused (error 111)
01:38:12.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:12.202507 22045 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } attempt: 2
01:38:12.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:12.208356 22045 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.111.252:39629: Network error: Client connection negotiation failed: client connection to 127.21.111.252:39629: connect: Connection refused (error 111)
01:38:12.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:12.282501 22125 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } attempt: 5
01:38:12.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:12.286064 22125 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.111.252:39629: Network error: Client connection negotiation failed: client connection to 127.21.111.252:39629: connect: Connection refused (error 111)
01:38:12.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:12.293934 22045 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } attempt: 3
01:38:12.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:12.297840 22045 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.111.252:39629: Network error: Client connection negotiation failed: client connection to 127.21.111.252:39629: connect: Connection refused (error 111)
01:38:12.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:12.438416 22045 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } attempt: 4
01:38:12.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:12.442723 22045 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.111.252:39629: Network error: Client connection negotiation failed: client connection to 127.21.111.252:39629: connect: Connection refused (error 111)
01:38:12.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:12.726279 22045 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } attempt: 5
01:38:12.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:12.730430 22045 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.111.252:39629: Network error: Client connection negotiation failed: client connection to 127.21.111.252:39629: connect: Connection refused (error 111)
01:38:12.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:12.833653 22125 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } attempt: 6
01:38:12.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:12.840217 22125 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.111.252:39629: Network error: Client connection negotiation failed: client connection to 127.21.111.252:39629: connect: Connection refused (error 111)
01:38:13.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:11.972137 22142 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:38:13.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:11.973199 22139 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:38:13.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:13.202091 22141 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
01:38:13.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.202208 22131 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:38:13.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.206123 22131 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:38:13.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.209263 22131 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:38:13.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.210652 22131 hybrid_clock.cc:648] HybridClock initialized: now 1790473093210591 us; error 66 us; skew 500 ppm
01:38:13.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.211437 22131 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:38:13.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 01:38:13 dist-test-slave-37dq krb5kdc[21962](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790473093, etypes {rep=17 tkt=17 ses=17}, kudu/127.21.111.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:38:13.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.217547 22131 init.cc:377] Logged in from keytab as kudu/127.21.111.252@KRBTEST.COM (short username kudu)
01:38:13.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.223196 22131 webserver.cc:460] Webserver started at http://127.21.111.252:37363/ using document root <none> and password file <none>
01:38:13.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.224085 22131 fs_manager.cc:359] Metadata directory not provided
01:38:13.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.224305 22131 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:38:13.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.224792 22131 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:38:13.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.229727 22131 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-2/data/instance:
01:38:13.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f01dea69c1d44542bde61d6b1f657868"
01:38:13.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 01:38:13 on dist-test-slave-37dq"
01:38:13.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.230930 22131 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-2/wal/instance:
01:38:13.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f01dea69c1d44542bde61d6b1f657868"
01:38:13.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 01:38:13 on dist-test-slave-37dq"
01:38:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.237877 22131 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.004s sys 0.004s
01:38:13.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.243040 22155 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:38:13.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.244050 22131 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
01:38:13.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.244316 22131 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-2/data,/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-2/wal
01:38:13.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f01dea69c1d44542bde61d6b1f657868"
01:38:13.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 01:38:13 on dist-test-slave-37dq"
01:38:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.244664 22131 fs_report.cc:383] FS layout report
01:38:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:38:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-2/wal
01:38:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-2/wal
01:38:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-2/data/data
01:38:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:38:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:38:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:38:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:38:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:38:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:38:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:38:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:38:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:38:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:38:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:38:13.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.282805 22045 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } attempt: 6
01:38:13.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.286675 22131 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:38:13.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:13.289013 22045 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.111.252:39629: Network error: Client connection negotiation failed: client connection to 127.21.111.252:39629: connect: Connection refused (error 111)
01:38:13.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.292666 22131 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:38:13.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.293139 22131 kserver.cc:163] Server-wide thread pool size limit: 3276
01:38:13.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.365988 22131 rpc_server.cc:307] RPC server started. Bound to: 127.21.111.252:39629
01:38:13.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.366132 22208 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.111.252:39629 every 8 connection(s)
01:38:13.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.368955 22131 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-2/data/info.pb
01:38:13.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.374315 22209 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:38:13.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.374576 21951 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu as pid 22131
01:38:13.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.375044 21951 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-2/wal/instance
01:38:13.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.390334 22209 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.111.254" port: 42895 } has no permanent_uuid. Determining permanent_uuid...
01:38:13.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 01:38:13 dist-test-slave-37dq krb5kdc[21962](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790473087, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.21.111.252@KRBTEST.COM
01:38:13.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 01:38:13 dist-test-slave-37dq krb5kdc[21962](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790473093, etypes {rep=17 tkt=17 ses=17}, kudu/127.21.111.252@KRBTEST.COM for kudu/127.21.111.254@KRBTEST.COM
01:38:13.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.437145 22209 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } has no permanent_uuid. Determining permanent_uuid...
01:38:13.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 01:38:13 dist-test-slave-37dq krb5kdc[21962](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790473087, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.21.111.253@KRBTEST.COM
01:38:13.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 01:38:13 dist-test-slave-37dq krb5kdc[21962](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790473093, etypes {rep=17 tkt=17 ses=17}, kudu/127.21.111.252@KRBTEST.COM for kudu/127.21.111.253@KRBTEST.COM
01:38:13.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.465849 22209 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } has no permanent_uuid. Determining permanent_uuid...
01:38:13.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 01:38:13 dist-test-slave-37dq krb5kdc[21962](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790473087, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.21.111.254@KRBTEST.COM
01:38:13.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 01:38:13 dist-test-slave-37dq krb5kdc[21962](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790473093, etypes {rep=17 tkt=17 ses=17}, kudu/127.21.111.252@KRBTEST.COM for kudu/127.21.111.252@KRBTEST.COM
01:38:13.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.493423 21951 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
01:38:13.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.496541 22209 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868: Bootstrap starting.
01:38:13.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.503433 22209 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868: Neither blocks nor log segments found. Creating new log.
01:38:13.503 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:38:13.504 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
01:38:13.505 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
01:38:13.505 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
01:38:13.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.505417 22209 log.cc:824] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868: Log is configured to *not* fsync() on all Append() calls
01:38:13.506 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.21.111.254"
port: 42895
}
bound_http_address {
host: "127.21.111.254"
port: 45347
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.21.111.253"
port: 37139
}
bound_http_address {
host: "127.21.111.253"
port: 41045
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.21.111.252"
port: 39629
}
bound_http_address {
host: "127.21.111.252"
port: 37363
}
}
}
01:38:13.507 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
01:38:13.507 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:38:13.508 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
01:38:13.508 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
01:38:13.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.510732 22209 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868: No bootstrap required, opened a new log
01:38:13.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.533900 22209 raft_consensus.cc:348] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a0e4466388e452592121c40b5e26931" member_type: VOTER last_known_addr { host: "127.21.111.254" port: 42895 } } peers { permanent_uuid: "0fd65b39ee41409a85c6a728d91f4051" member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } } peers { permanent_uuid: "f01dea69c1d44542bde61d6b1f657868" member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } }
01:38:13.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.534761 22209 raft_consensus.cc:374] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:38:13.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.535064 22209 raft_consensus.cc:729] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f01dea69c1d44542bde61d6b1f657868, State: Initialized, Role: FOLLOWER
01:38:13.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 01:38:13 dist-test-slave-37dq krb5kdc[21962](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790473093, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:38:13.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
01:38:13.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.536073 22209 consensus_queue.cc:260] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a0e4466388e452592121c40b5e26931" member_type: VOTER last_known_addr { host: "127.21.111.254" port: 42895 } } peers { permanent_uuid: "0fd65b39ee41409a85c6a728d91f4051" member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } } peers { permanent_uuid: "f01dea69c1d44542bde61d6b1f657868" member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } }
01:38:13.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.539023 22220 sys_catalog.cc:455] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a0e4466388e452592121c40b5e26931" member_type: VOTER last_known_addr { host: "127.21.111.254" port: 42895 } } peers { permanent_uuid: "0fd65b39ee41409a85c6a728d91f4051" member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } } peers { permanent_uuid: "f01dea69c1d44542bde61d6b1f657868" member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } } }
01:38:13.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.540053 22220 sys_catalog.cc:458] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868 [sys.catalog]: This master's current role is: FOLLOWER
01:38:13.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.540850 22209 sys_catalog.cc:564] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868 [sys.catalog]: configured and running, proceeding with master startup.
01:38:13.544 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:38:13.546 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/krb5kdc/krb5cc
01:38:13.547 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
01:38:13.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:13.548055 22163 tablet.cc:2369] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868: Can't schedule compaction. Clean time has not been advanced past its initial value.
01:38:13.548 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-user
01:38:13.549 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-user"
}
01:38:13.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:13.576668 22233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:38:13.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:13.576928 22233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:38:13.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 01:38:13 dist-test-slave-37dq krb5kdc[21962](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790473093, etypes {rep=17 tkt=17 ses=17}, test-user@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:38:13.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-user@KRBTEST.COM:
01:38:13.585 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:38:13.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.572366 22131 master_runner.cc:186] Error getting master registration for 127.21.111.254:42895: OK, instance_id { permanent_uuid: "9a0e4466388e452592121c40b5e26931" instance_seqno: 1790473089649976 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
01:38:13.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.605136 22131 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:38:13.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.900816 22125 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } attempt: 7
01:38:13.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 01:38:13 dist-test-slave-37dq krb5kdc[21962](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790473091, etypes {rep=17 tkt=17 ses=17}, kudu/127.21.111.253@KRBTEST.COM for kudu/127.21.111.252@KRBTEST.COM
01:38:13.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.933514 22125 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051: Bootstrap starting.
01:38:13.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.938359 22125 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051: Neither blocks nor log segments found. Creating new log.
01:38:13.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.939903 22125 log.cc:824] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051: Log is configured to *not* fsync() on all Append() calls
01:38:13.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.944844 22125 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051: No bootstrap required, opened a new log
01:38:13.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.969565 22125 raft_consensus.cc:348] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a0e4466388e452592121c40b5e26931" member_type: VOTER last_known_addr { host: "127.21.111.254" port: 42895 } } peers { permanent_uuid: "0fd65b39ee41409a85c6a728d91f4051" member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } } peers { permanent_uuid: "f01dea69c1d44542bde61d6b1f657868" member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } }
01:38:13.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.970214 22125 raft_consensus.cc:374] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:38:13.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.970465 22125 raft_consensus.cc:729] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0fd65b39ee41409a85c6a728d91f4051, State: Initialized, Role: FOLLOWER
01:38:13.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.971210 22125 consensus_queue.cc:260] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a0e4466388e452592121c40b5e26931" member_type: VOTER last_known_addr { host: "127.21.111.254" port: 42895 } } peers { permanent_uuid: "0fd65b39ee41409a85c6a728d91f4051" member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } } peers { permanent_uuid: "f01dea69c1d44542bde61d6b1f657868" member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } }
01:38:13.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.973757 22239 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a0e4466388e452592121c40b5e26931" member_type: VOTER last_known_addr { host: "127.21.111.254" port: 42895 } } peers { permanent_uuid: "0fd65b39ee41409a85c6a728d91f4051" member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } } peers { permanent_uuid: "f01dea69c1d44542bde61d6b1f657868" member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } } }
01:38:13.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.974500 22239 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [sys.catalog]: This master's current role is: FOLLOWER
01:38:13.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.975374 22125 sys_catalog.cc:564] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [sys.catalog]: configured and running, proceeding with master startup.
01:38:14.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:14.003316 22250 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:38:14.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:14.003610 22250 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:38:14.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.010617 22220 raft_consensus.cc:482] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:38:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.011044 22220 raft_consensus.cc:504] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a0e4466388e452592121c40b5e26931" member_type: VOTER last_known_addr { host: "127.21.111.254" port: 42895 } } peers { permanent_uuid: "0fd65b39ee41409a85c6a728d91f4051" member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } } peers { permanent_uuid: "f01dea69c1d44542bde61d6b1f657868" member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } }
01:38:14.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.013618 22220 leader_election.cc:288] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9a0e4466388e452592121c40b5e26931 (127.21.111.254:42895), 0fd65b39ee41409a85c6a728d91f4051 (127.21.111.253:37139)
01:38:14.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.016613 22100 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f01dea69c1d44542bde61d6b1f657868" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0fd65b39ee41409a85c6a728d91f4051" is_pre_election: true
01:38:14.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.017464 22100 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f01dea69c1d44542bde61d6b1f657868 in term 0.
01:38:14.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.018836 22158 leader_election.cc:302] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0fd65b39ee41409a85c6a728d91f4051, f01dea69c1d44542bde61d6b1f657868; no voters:
01:38:14.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.019745 22220 raft_consensus.cc:2781] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868 [term 0 FOLLOWER]: Leader pre-election won for term 1
01:38:14.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.020056 22220 raft_consensus.cc:482] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
01:38:14.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.020377 22220 raft_consensus.cc:3037] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868 [term 0 FOLLOWER]: Advancing to term 1
01:38:14.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.029543 22220 raft_consensus.cc:504] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a0e4466388e452592121c40b5e26931" member_type: VOTER last_known_addr { host: "127.21.111.254" port: 42895 } } peers { permanent_uuid: "0fd65b39ee41409a85c6a728d91f4051" member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } } peers { permanent_uuid: "f01dea69c1d44542bde61d6b1f657868" member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } }
01:38:14.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:13.999847 22051 master_runner.cc:186] Error getting master registration for 127.21.111.254:42895: OK, instance_id { permanent_uuid: "9a0e4466388e452592121c40b5e26931" instance_seqno: 1790473089649976 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
01:38:14.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.035086 22051 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:38:14.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.035251 22100 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f01dea69c1d44542bde61d6b1f657868" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0fd65b39ee41409a85c6a728d91f4051"
01:38:14.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.035821 22100 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [term 0 FOLLOWER]: Advancing to term 1
01:38:14.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.039914 22220 leader_election.cc:288] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868 [CANDIDATE]: Term 1 election: Requested vote from peers 9a0e4466388e452592121c40b5e26931 (127.21.111.254:42895), 0fd65b39ee41409a85c6a728d91f4051 (127.21.111.253:37139)
01:38:14.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.042119 22100 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f01dea69c1d44542bde61d6b1f657868 in term 1.
01:38:14.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.043111 22158 leader_election.cc:302] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0fd65b39ee41409a85c6a728d91f4051, f01dea69c1d44542bde61d6b1f657868; no voters:
01:38:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.043968 22220 raft_consensus.cc:2781] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868 [term 1 FOLLOWER]: Leader election won for term 1
01:38:14.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.046985 22220 raft_consensus.cc:686] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868 [term 1 LEADER]: Becoming Leader. State: Replica: f01dea69c1d44542bde61d6b1f657868, State: Running, Role: LEADER
01:38:14.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.048961 22220 consensus_queue.cc:237] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a0e4466388e452592121c40b5e26931" member_type: VOTER last_known_addr { host: "127.21.111.254" port: 42895 } } peers { permanent_uuid: "0fd65b39ee41409a85c6a728d91f4051" member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } } peers { permanent_uuid: "f01dea69c1d44542bde61d6b1f657868" member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } }
01:38:14.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.062783 22252 sys_catalog.cc:455] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f01dea69c1d44542bde61d6b1f657868. Latest consensus state: current_term: 1 leader_uuid: "f01dea69c1d44542bde61d6b1f657868" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a0e4466388e452592121c40b5e26931" member_type: VOTER last_known_addr { host: "127.21.111.254" port: 42895 } } peers { permanent_uuid: "0fd65b39ee41409a85c6a728d91f4051" member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } } peers { permanent_uuid: "f01dea69c1d44542bde61d6b1f657868" member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } } }
01:38:14.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.063534 22252 sys_catalog.cc:458] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868 [sys.catalog]: This master's current role is: LEADER
01:38:14.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.066903 22254 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
01:38:14.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.083637 22254 catalog_manager.cc:1484] Initializing Kudu cluster ID...
01:38:14.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.119264 22100 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [term 1 FOLLOWER]: Refusing update from remote peer f01dea69c1d44542bde61d6b1f657868: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
01:38:14.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.121273 22252 consensus_queue.cc:1035] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0fd65b39ee41409a85c6a728d91f4051" member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
01:38:14.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.177098 22239 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f01dea69c1d44542bde61d6b1f657868. Latest consensus state: current_term: 1 leader_uuid: "f01dea69c1d44542bde61d6b1f657868" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a0e4466388e452592121c40b5e26931" member_type: VOTER last_known_addr { host: "127.21.111.254" port: 42895 } } peers { permanent_uuid: "0fd65b39ee41409a85c6a728d91f4051" member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } } peers { permanent_uuid: "f01dea69c1d44542bde61d6b1f657868" member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } } }
01:38:14.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.178143 22239 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [sys.catalog]: This master's current role is: FOLLOWER
01:38:14.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.187389 22220 sys_catalog.cc:455] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "f01dea69c1d44542bde61d6b1f657868" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a0e4466388e452592121c40b5e26931" member_type: VOTER last_known_addr { host: "127.21.111.254" port: 42895 } } peers { permanent_uuid: "0fd65b39ee41409a85c6a728d91f4051" member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } } peers { permanent_uuid: "f01dea69c1d44542bde61d6b1f657868" member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } } }
01:38:14.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.188165 22220 sys_catalog.cc:458] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868 [sys.catalog]: This master's current role is: LEADER
01:38:14.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.190546 22239 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "f01dea69c1d44542bde61d6b1f657868" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a0e4466388e452592121c40b5e26931" member_type: VOTER last_known_addr { host: "127.21.111.254" port: 42895 } } peers { permanent_uuid: "0fd65b39ee41409a85c6a728d91f4051" member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } } peers { permanent_uuid: "f01dea69c1d44542bde61d6b1f657868" member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } } }
01:38:14.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.191283 22239 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [sys.catalog]: This master's current role is: FOLLOWER
01:38:14.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.194610 22254 catalog_manager.cc:1347] Generated new cluster ID: ba1632ff75af4af6ac42277a5ebc6ae1
01:38:14.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.194898 22254 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
01:38:14.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.233595 22254 catalog_manager.cc:1370] Generated new certificate authority record
01:38:14.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.238468 22254 catalog_manager.cc:1504] Loading token signing keys...
01:38:14.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.277010 22254 catalog_manager.cc:5868] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868: Generated new TSK 0
01:38:14.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.278159 22254 catalog_manager.cc:1514] Initializing in-progress tserver states...
01:38:14.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.322386 22045 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } attempt: 7
01:38:14.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 01:38:14 dist-test-slave-37dq krb5kdc[21962](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790473089, etypes {rep=17 tkt=17 ses=17}, kudu/127.21.111.254@KRBTEST.COM for kudu/127.21.111.252@KRBTEST.COM
01:38:14.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.375983 22045 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9a0e4466388e452592121c40b5e26931: Bootstrap starting.
01:38:14.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.391717 22045 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 9a0e4466388e452592121c40b5e26931: Neither blocks nor log segments found. Creating new log.
01:38:14.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.397441 22045 log.cc:824] T 00000000000000000000000000000000 P 9a0e4466388e452592121c40b5e26931: Log is configured to *not* fsync() on all Append() calls
01:38:14.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.419811 22045 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9a0e4466388e452592121c40b5e26931: No bootstrap required, opened a new log
01:38:14.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.500142 22045 raft_consensus.cc:348] T 00000000000000000000000000000000 P 9a0e4466388e452592121c40b5e26931 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a0e4466388e452592121c40b5e26931" member_type: VOTER last_known_addr { host: "127.21.111.254" port: 42895 } } peers { permanent_uuid: "0fd65b39ee41409a85c6a728d91f4051" member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } } peers { permanent_uuid: "f01dea69c1d44542bde61d6b1f657868" member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } }
01:38:14.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.501190 22045 raft_consensus.cc:374] T 00000000000000000000000000000000 P 9a0e4466388e452592121c40b5e26931 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:38:14.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.501562 22045 raft_consensus.cc:729] T 00000000000000000000000000000000 P 9a0e4466388e452592121c40b5e26931 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9a0e4466388e452592121c40b5e26931, State: Initialized, Role: FOLLOWER
01:38:14.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.502527 22045 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9a0e4466388e452592121c40b5e26931 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a0e4466388e452592121c40b5e26931" member_type: VOTER last_known_addr { host: "127.21.111.254" port: 42895 } } peers { permanent_uuid: "0fd65b39ee41409a85c6a728d91f4051" member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } } peers { permanent_uuid: "f01dea69c1d44542bde61d6b1f657868" member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } }
01:38:14.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.519464 22262 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9a0e4466388e452592121c40b5e26931 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a0e4466388e452592121c40b5e26931" member_type: VOTER last_known_addr { host: "127.21.111.254" port: 42895 } } peers { permanent_uuid: "0fd65b39ee41409a85c6a728d91f4051" member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } } peers { permanent_uuid: "f01dea69c1d44542bde61d6b1f657868" member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } } }
01:38:14.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.520344 22262 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9a0e4466388e452592121c40b5e26931 [sys.catalog]: This master's current role is: FOLLOWER
01:38:14.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.522001 22045 sys_catalog.cc:564] T 00000000000000000000000000000000 P 9a0e4466388e452592121c40b5e26931 [sys.catalog]: configured and running, proceeding with master startup.
01:38:14.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.522552 22018 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 9a0e4466388e452592121c40b5e26931 [term 0 FOLLOWER]: Advancing to term 1
01:38:14.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.524992 22020 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f01dea69c1d44542bde61d6b1f657868" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9a0e4466388e452592121c40b5e26931" is_pre_election: true
01:38:14.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.526329 22019 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f01dea69c1d44542bde61d6b1f657868" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9a0e4466388e452592121c40b5e26931"
01:38:14.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.531462 22018 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 9a0e4466388e452592121c40b5e26931 [term 1 FOLLOWER]: Refusing update from remote peer f01dea69c1d44542bde61d6b1f657868: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
01:38:14.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.532882 22252 consensus_queue.cc:1035] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9a0e4466388e452592121c40b5e26931" member_type: VOTER last_known_addr { host: "127.21.111.254" port: 42895 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
01:38:14.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.608125 22262 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9a0e4466388e452592121c40b5e26931 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f01dea69c1d44542bde61d6b1f657868. Latest consensus state: current_term: 1 leader_uuid: "f01dea69c1d44542bde61d6b1f657868" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a0e4466388e452592121c40b5e26931" member_type: VOTER last_known_addr { host: "127.21.111.254" port: 42895 } } peers { permanent_uuid: "0fd65b39ee41409a85c6a728d91f4051" member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } } peers { permanent_uuid: "f01dea69c1d44542bde61d6b1f657868" member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } } }
01:38:14.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.608223 22269 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9a0e4466388e452592121c40b5e26931 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "f01dea69c1d44542bde61d6b1f657868" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a0e4466388e452592121c40b5e26931" member_type: VOTER last_known_addr { host: "127.21.111.254" port: 42895 } } peers { permanent_uuid: "0fd65b39ee41409a85c6a728d91f4051" member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } } peers { permanent_uuid: "f01dea69c1d44542bde61d6b1f657868" member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } } }
01:38:14.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.608912 22262 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9a0e4466388e452592121c40b5e26931 [sys.catalog]: This master's current role is: FOLLOWER
01:38:14.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.609050 22269 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9a0e4466388e452592121c40b5e26931 [sys.catalog]: This master's current role is: FOLLOWER
01:38:14.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.637188 22279 catalog_manager.cc:1259] Loaded cluster ID: ba1632ff75af4af6ac42277a5ebc6ae1
01:38:14.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.637461 22279 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 9a0e4466388e452592121c40b5e26931: loading cluster ID for follower catalog manager: success
01:38:14.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.647894 22279 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 9a0e4466388e452592121c40b5e26931: acquiring CA information for follower catalog manager: success
01:38:14.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:14.654292 22279 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 9a0e4466388e452592121c40b5e26931: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
01:38:15.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:15.010021 22250 catalog_manager.cc:1259] Loaded cluster ID: ba1632ff75af4af6ac42277a5ebc6ae1
01:38:15.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:15.010547 22250 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051: loading cluster ID for follower catalog manager: success
01:38:15.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:15.019299 22250 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051: acquiring CA information for follower catalog manager: success
01:38:15.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:15.025811 22250 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
01:38:15.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:15.067075 22252 sys_catalog.cc:455] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "f01dea69c1d44542bde61d6b1f657868" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a0e4466388e452592121c40b5e26931" member_type: VOTER last_known_addr { host: "127.21.111.254" port: 42895 } } peers { permanent_uuid: "0fd65b39ee41409a85c6a728d91f4051" member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } } peers { permanent_uuid: "f01dea69c1d44542bde61d6b1f657868" member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } } }
01:38:15.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:15.069128 22252 sys_catalog.cc:458] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868 [sys.catalog]: This master's current role is: LEADER
01:38:16.586 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.21.111.253:37139
01:38:16.587 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
01:38:16.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:16.589342 21951 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu with pid 22051
01:38:16.621 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:38:16.621 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.21.111.254:42895
01:38:16.621 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
01:38:16.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:16.621747 21951 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu with pid 21976
01:38:16.651 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:38:16.651 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.21.111.252:39629
01:38:16.651 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
01:38:16.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:16.651772 21951 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu with pid 22131
01:38:16.680 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:38:16.680 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.21.111.253:37139
01:38:16.680 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
01:38:16.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:16.682518 21951 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu
01:38:16.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu
01:38:16.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-1/wal
01:38:16.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-1/data
01:38:16.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:38:16.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:38:16.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:38:16.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:38:16.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:38:16.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:38:16.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-1/logs
01:38:16.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-1/data/info.pb
01:38:16.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:38:16.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:38:16.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:38:16.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:38:16.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:38:16.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:38:16.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:38:16.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:38:16.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:38:16.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:38:16.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:38:16.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:38:16.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.111.253:37139
01:38:16.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.111.253
01:38:16.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41045
01:38:16.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/krb5kdc/kudu_127.21.111.253.keytab
01:38:16.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.21.111.253
01:38:16.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:38:16.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:38:16.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:38:16.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.111.212:37441
01:38:16.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:38:16.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:38:16.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:38:16.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:38:16.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.111.254:42895,127.21.111.253:37139,127.21.111.252:39629
01:38:16.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:38:16.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46859/jwks.json
01:38:16.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:38:16.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:16.988153 22289 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:38:16.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:16.988721 22289 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:38:16.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:16.989288 22289 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:38:17.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:17.021059 22289 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:38:17.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:17.021381 22289 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:38:17.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:17.021703 22289 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:38:17.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:17.021938 22289 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:38:17.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:17.022149 22289 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:38:17.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:17.022361 22289 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:38:17.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:17.022519 22289 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46859/jwks.json
01:38:17.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:17.062040 22289 master_runner.cc:386] Master server non-default flags:
01:38:17.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:38:17.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.111.212:37441
01:38:17.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:38:17.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:38:17.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-1/data
01:38:17.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-1/wal
01:38:17.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:38:17.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:38:17.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.111.254:42895,127.21.111.253:37139,127.21.111.252:39629
01:38:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/krb5kdc/kudu_127.21.111.253.keytab
01:38:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.21.111.253
01:38:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:38:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:38:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:38:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.111.253:37139
01:38:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:38:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:38:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:38:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46859/jwks.json
01:38:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:38:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:38:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:38:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:38:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:38:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-1/data/info.pb
01:38:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.111.253
01:38:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41045
01:38:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:38:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:38:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:38:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:38:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:38:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-1/logs
01:38:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:38:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:38:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:38:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:38:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:38:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:38:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:38:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 00:02:55 UTC on ccycloud.quasar-qgnmee.root.comops.site
01:38:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675483
01:38:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:38:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:17.063175 22289 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:38:17.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:17.064101 22289 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:38:17.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:17.082039 22295 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:38:18.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:17.082535 22296 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:38:18.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:17.082883 22298 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:38:18.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:17.084894 22289 server_base.cc:1034] running on GCE node
01:38:18.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:18.303017 22289 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:38:18.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:18.305611 22289 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:38:18.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:18.306953 22289 hybrid_clock.cc:648] HybridClock initialized: now 1790473098306928 us; error 38 us; skew 500 ppm
01:38:18.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:18.307713 22289 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:38:18.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 01:38:18 dist-test-slave-37dq krb5kdc[21962](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790473098, etypes {rep=17 tkt=17 ses=17}, kudu/127.21.111.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:38:18.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:18.313531 22289 init.cc:377] Logged in from keytab as kudu/127.21.111.253@KRBTEST.COM (short username kudu)
01:38:18.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:18.318624 22289 webserver.cc:460] Webserver started at http://127.21.111.253:41045/ using document root <none> and password file <none>
01:38:18.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:18.319533 22289 fs_manager.cc:359] Metadata directory not provided
01:38:18.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:18.319725 22289 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:38:18.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:18.327641 22289 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.006s sys 0.000s
01:38:18.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:18.331912 22307 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:38:18.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:18.332942 22289 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
01:38:18.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:18.333302 22289 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-1/data,/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-1/wal
01:38:18.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0fd65b39ee41409a85c6a728d91f4051"
01:38:18.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 01:38:11 on dist-test-slave-37dq"
01:38:18.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:18.335595 22289 fs_report.cc:383] FS layout report
01:38:18.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:38:18.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-1/wal
01:38:18.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-1/wal
01:38:18.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-1/data/data
01:38:18.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:38:18.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:38:18.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:38:18.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:38:18.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:38:18.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:38:18.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:38:18.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:38:18.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:38:18.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:38:18.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:38:18.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:18.368427 22289 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:38:18.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:18.374658 22289 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:38:18.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:18.375171 22289 kserver.cc:163] Server-wide thread pool size limit: 3276
01:38:18.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:18.456055 22289 rpc_server.cc:307] RPC server started. Bound to: 127.21.111.253:37139
01:38:18.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:18.456168 22358 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.111.253:37139 every 8 connection(s)
01:38:18.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:18.459112 22289 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-1/data/info.pb
01:38:18.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:18.468389 21951 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu as pid 22289
01:38:18.469 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:38:18.469 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.21.111.254:42895
01:38:18.469 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
01:38:18.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:18.469527 22359 sys_catalog.cc:263] Verifying existing consensus state
01:38:18.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:18.470788 21951 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu
01:38:18.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu
01:38:18.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-0/wal
01:38:18.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-0/data
01:38:18.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:38:18.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:38:18.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:38:18.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:38:18.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:38:18.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:38:18.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-0/logs
01:38:18.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-0/data/info.pb
01:38:18.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:38:18.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:38:18.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:38:18.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:38:18.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:38:18.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:38:18.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:38:18.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:38:18.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:38:18.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:38:18.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:38:18.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:38:18.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.111.254:42895
01:38:18.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.111.254
01:38:18.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45347
01:38:18.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/krb5kdc/kudu_127.21.111.254.keytab
01:38:18.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.21.111.254
01:38:18.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:38:18.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:38:18.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:38:18.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.111.212:37441
01:38:18.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:38:18.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:38:18.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:38:18.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:38:18.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.111.254:42895,127.21.111.253:37139,127.21.111.252:39629
01:38:18.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:38:18.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46859/jwks.json
01:38:18.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:38:18.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:18.474817 22359 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051: Bootstrap starting.
01:38:18.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:18.513640 22359 log.cc:824] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051: Log is configured to *not* fsync() on all Append() calls
01:38:18.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:18.527920 22359 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
01:38:18.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:18.528746 22359 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051: Bootstrap complete.
01:38:18.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:18.555488 22359 raft_consensus.cc:348] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a0e4466388e452592121c40b5e26931" member_type: VOTER last_known_addr { host: "127.21.111.254" port: 42895 } } peers { permanent_uuid: "0fd65b39ee41409a85c6a728d91f4051" member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } } peers { permanent_uuid: "f01dea69c1d44542bde61d6b1f657868" member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } }
01:38:18.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:18.558120 22359 raft_consensus.cc:729] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0fd65b39ee41409a85c6a728d91f4051, State: Initialized, Role: FOLLOWER
01:38:18.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:18.558971 22359 consensus_queue.cc:260] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a0e4466388e452592121c40b5e26931" member_type: VOTER last_known_addr { host: "127.21.111.254" port: 42895 } } peers { permanent_uuid: "0fd65b39ee41409a85c6a728d91f4051" member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } } peers { permanent_uuid: "f01dea69c1d44542bde61d6b1f657868" member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } }
01:38:18.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:18.562057 22363 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a0e4466388e452592121c40b5e26931" member_type: VOTER last_known_addr { host: "127.21.111.254" port: 42895 } } peers { permanent_uuid: "0fd65b39ee41409a85c6a728d91f4051" member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } } peers { permanent_uuid: "f01dea69c1d44542bde61d6b1f657868" member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } } }
01:38:18.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:18.563179 22363 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [sys.catalog]: This master's current role is: FOLLOWER
01:38:18.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:18.563383 22359 sys_catalog.cc:564] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [sys.catalog]: configured and running, proceeding with master startup.
01:38:18.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:18.596721 22374 catalog_manager.cc:1259] Loaded cluster ID: ba1632ff75af4af6ac42277a5ebc6ae1
01:38:18.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:18.597162 22374 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051: loading cluster ID for follower catalog manager: success
01:38:18.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:18.605381 22374 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051: acquiring CA information for follower catalog manager: success
01:38:18.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:18.611021 22374 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
01:38:18.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:18.592597 22289 master_runner.cc:186] Error getting master registration for 127.21.111.254:42895: Network error: Client connection negotiation failed: client connection to 127.21.111.254:42895: connect: Connection refused (error 111),
01:38:18.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:18.623736 22289 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:38:18.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:18.792960 22361 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:38:18.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:18.793468 22361 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:38:18.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:18.794117 22361 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:38:18.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:18.825459 22361 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:38:18.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:18.825695 22361 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:38:18.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:18.826018 22361 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:38:18.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:18.826226 22361 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:38:18.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:18.826417 22361 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:38:18.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:18.826599 22361 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:38:18.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:18.826733 22361 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46859/jwks.json
01:38:18.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:18.864059 22361 master_runner.cc:386] Master server non-default flags:
01:38:18.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:38:18.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.111.212:37441
01:38:18.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:38:18.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:38:18.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-0/data
01:38:18.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-0/wal
01:38:18.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:38:18.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:38:18.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.111.254:42895,127.21.111.253:37139,127.21.111.252:39629
01:38:18.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/krb5kdc/kudu_127.21.111.254.keytab
01:38:18.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.21.111.254
01:38:18.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:38:18.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:38:18.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:38:18.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.111.254:42895
01:38:18.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:38:18.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:38:18.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:38:18.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46859/jwks.json
01:38:18.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:38:18.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:38:18.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:38:18.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:38:18.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:38:18.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-0/data/info.pb
01:38:18.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.111.254
01:38:18.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45347
01:38:18.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:38:18.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:38:18.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:38:18.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:38:18.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:38:18.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-0/logs
01:38:18.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:38:18.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:38:18.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:38:18.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:38:18.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:38:18.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:38:18.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:38:18.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 00:02:55 UTC on ccycloud.quasar-qgnmee.root.comops.site
01:38:18.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675483
01:38:18.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:38:18.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:18.865487 22361 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:38:18.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:18.866482 22361 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:38:18.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:18.890377 22382 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:38:20.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:20.309630 22387 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:38:20.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:20.310365 22387 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a0e4466388e452592121c40b5e26931" member_type: VOTER last_known_addr { host: "127.21.111.254" port: 42895 } } peers { permanent_uuid: "0fd65b39ee41409a85c6a728d91f4051" member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } } peers { permanent_uuid: "f01dea69c1d44542bde61d6b1f657868" member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } }
01:38:20.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:20.318588 22310 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.111.252:39629: connect: Connection refused (error 111)
01:38:20.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:20.324617 22310 leader_election.cc:334] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer f01dea69c1d44542bde61d6b1f657868 (127.21.111.252:39629): Network error: Client connection negotiation failed: client connection to 127.21.111.252:39629: connect: Connection refused (error 111)
01:38:20.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:20.327931 22311 leader_election.cc:334] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 9a0e4466388e452592121c40b5e26931 (127.21.111.254:42895): Network error: Client connection negotiation failed: client connection to 127.21.111.254:42895: connect: Connection refused (error 111)
01:38:20.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:20.328480 22311 leader_election.cc:302] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0fd65b39ee41409a85c6a728d91f4051; no voters: 9a0e4466388e452592121c40b5e26931, f01dea69c1d44542bde61d6b1f657868
01:38:20.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:20.331463 22387 leader_election.cc:288] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 9a0e4466388e452592121c40b5e26931 (127.21.111.254:42895), f01dea69c1d44542bde61d6b1f657868 (127.21.111.252:39629)
01:38:20.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:20.332166 22387 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
01:38:20.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:20.284745 22381 debug-util.cc:398] Leaking SignalData structure 0x7b0800014920 after lost signal to thread 22361
01:38:20.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:20.401116 22361 thread.cc:636] GCE (cloud detector) Time spent creating pthread: real 1.512s user 0.511s sys 0.982s
01:38:20.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:20.405627 22361 thread.cc:602] GCE (cloud detector) Time spent starting thread: real 1.517s user 0.512s sys 0.983s
01:38:20.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:18.891942 22383 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:38:20.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:20.416425 22393 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:38:20.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:20.416934 22361 server_base.cc:1034] running on GCE node
01:38:20.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:20.418231 22361 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:38:20.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:20.421144 22361 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:38:20.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:20.422611 22361 hybrid_clock.cc:648] HybridClock initialized: now 1790473100422491 us; error 80 us; skew 500 ppm
01:38:20.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:20.423691 22361 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:38:20.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 01:38:20 dist-test-slave-37dq krb5kdc[21962](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790473100, etypes {rep=17 tkt=17 ses=17}, kudu/127.21.111.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:38:20.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:20.434168 22361 init.cc:377] Logged in from keytab as kudu/127.21.111.254@KRBTEST.COM (short username kudu)
01:38:20.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:20.439401 22361 webserver.cc:460] Webserver started at http://127.21.111.254:45347/ using document root <none> and password file <none>
01:38:20.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:20.440227 22361 fs_manager.cc:359] Metadata directory not provided
01:38:20.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:20.440418 22361 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:38:20.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:20.448371 22361 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.007s sys 0.000s
01:38:20.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:20.453512 22399 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:38:20.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:20.454922 22361 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
01:38:20.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:20.455298 22361 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-0/data,/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-0/wal
01:38:20.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9a0e4466388e452592121c40b5e26931"
01:38:20.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 01:38:09 on dist-test-slave-37dq"
01:38:20.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:20.458621 22361 fs_report.cc:383] FS layout report
01:38:20.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:38:20.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-0/wal
01:38:20.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-0/wal
01:38:20.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-0/data/data
01:38:20.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:38:20.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:38:20.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:38:20.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:38:20.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:38:20.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:38:20.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:38:20.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:38:20.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:38:20.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:38:20.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:38:20.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:20.522657 22361 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:38:20.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:20.528344 22361 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:38:20.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:20.528818 22361 kserver.cc:163] Server-wide thread pool size limit: 3276
01:38:20.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:20.615289 22361 rpc_server.cc:307] RPC server started. Bound to: 127.21.111.254:42895
01:38:20.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:20.615520 22450 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.111.254:42895 every 8 connection(s)
01:38:20.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:20.618381 22361 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-0/data/info.pb
01:38:20.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:20.625415 21951 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu as pid 22361
01:38:20.626 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:38:20.626 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.21.111.252:39629
01:38:20.626 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
01:38:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:20.627961 21951 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu
01:38:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu
01:38:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-2/wal
01:38:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-2/data
01:38:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:38:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:38:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:38:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:38:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:38:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:38:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-2/logs
01:38:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-2/data/info.pb
01:38:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:38:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:38:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:38:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:38:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:38:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:38:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:38:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:38:20.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:38:20.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:38:20.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:38:20.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:38:20.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.111.252:39629
01:38:20.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.111.252
01:38:20.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37363
01:38:20.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/krb5kdc/kudu_127.21.111.252.keytab
01:38:20.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.21.111.252
01:38:20.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:38:20.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:38:20.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:38:20.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.111.212:37441
01:38:20.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:38:20.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:38:20.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:38:20.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:38:20.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.111.254:42895,127.21.111.253:37139,127.21.111.252:39629
01:38:20.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:38:20.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46859/jwks.json
01:38:20.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:38:20.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:20.629993 22451 sys_catalog.cc:263] Verifying existing consensus state
01:38:20.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:20.635965 22451 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9a0e4466388e452592121c40b5e26931: Bootstrap starting.
01:38:20.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:20.675496 22451 log.cc:824] T 00000000000000000000000000000000 P 9a0e4466388e452592121c40b5e26931: Log is configured to *not* fsync() on all Append() calls
01:38:20.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:20.691491 22451 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9a0e4466388e452592121c40b5e26931: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
01:38:20.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:20.692456 22451 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9a0e4466388e452592121c40b5e26931: Bootstrap complete.
01:38:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:20.717343 22451 raft_consensus.cc:348] T 00000000000000000000000000000000 P 9a0e4466388e452592121c40b5e26931 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a0e4466388e452592121c40b5e26931" member_type: VOTER last_known_addr { host: "127.21.111.254" port: 42895 } } peers { permanent_uuid: "0fd65b39ee41409a85c6a728d91f4051" member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } } peers { permanent_uuid: "f01dea69c1d44542bde61d6b1f657868" member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } }
01:38:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:20.719585 22451 raft_consensus.cc:729] T 00000000000000000000000000000000 P 9a0e4466388e452592121c40b5e26931 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9a0e4466388e452592121c40b5e26931, State: Initialized, Role: FOLLOWER
01:38:20.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:20.720481 22451 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9a0e4466388e452592121c40b5e26931 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a0e4466388e452592121c40b5e26931" member_type: VOTER last_known_addr { host: "127.21.111.254" port: 42895 } } peers { permanent_uuid: "0fd65b39ee41409a85c6a728d91f4051" member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } } peers { permanent_uuid: "f01dea69c1d44542bde61d6b1f657868" member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } }
01:38:20.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:20.725571 22451 sys_catalog.cc:564] T 00000000000000000000000000000000 P 9a0e4466388e452592121c40b5e26931 [sys.catalog]: configured and running, proceeding with master startup.
01:38:20.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:20.723698 22455 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9a0e4466388e452592121c40b5e26931 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a0e4466388e452592121c40b5e26931" member_type: VOTER last_known_addr { host: "127.21.111.254" port: 42895 } } peers { permanent_uuid: "0fd65b39ee41409a85c6a728d91f4051" member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } } peers { permanent_uuid: "f01dea69c1d44542bde61d6b1f657868" member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } } }
01:38:20.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:20.726372 22455 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9a0e4466388e452592121c40b5e26931 [sys.catalog]: This master's current role is: FOLLOWER
01:38:20.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:20.783215 22466 catalog_manager.cc:1259] Loaded cluster ID: ba1632ff75af4af6ac42277a5ebc6ae1
01:38:20.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:20.785674 22466 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 9a0e4466388e452592121c40b5e26931: loading cluster ID for follower catalog manager: success
01:38:20.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:20.801692 22466 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 9a0e4466388e452592121c40b5e26931: acquiring CA information for follower catalog manager: success
01:38:20.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 01:38:20 dist-test-slave-37dq krb5kdc[21962](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790473100, etypes {rep=17 tkt=17 ses=17}, kudu/127.21.111.254@KRBTEST.COM for kudu/127.21.111.254@KRBTEST.COM
01:38:20.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:20.828397 22466 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 9a0e4466388e452592121c40b5e26931: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
01:38:20.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:20.881778 22361 master_runner.cc:186] Error getting master registration for 127.21.111.252:39629: Network error: Client connection negotiation failed: client connection to 127.21.111.252:39629: connect: Connection refused (error 111),
01:38:20.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:20.921778 22361 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:38:21.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:21.109074 22453 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:38:21.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:21.109820 22453 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:38:21.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:21.110821 22453 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:38:21.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:21.164779 22453 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:38:21.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:21.165061 22453 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:38:21.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:21.165482 22453 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:38:21.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:21.165822 22453 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:38:21.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:21.166146 22453 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:38:21.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:21.166467 22453 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:38:21.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:21.166702 22453 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46859/jwks.json
01:38:21.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:21.229406 22453 master_runner.cc:386] Master server non-default flags:
01:38:21.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:38:21.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.111.212:37441
01:38:21.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:38:21.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:38:21.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-2/data
01:38:21.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-2/wal
01:38:21.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:38:21.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:38:21.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.111.254:42895,127.21.111.253:37139,127.21.111.252:39629
01:38:21.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/krb5kdc/kudu_127.21.111.252.keytab
01:38:21.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.21.111.252
01:38:21.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:38:21.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:38:21.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:38:21.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.111.252:39629
01:38:21.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:38:21.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:38:21.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:38:21.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46859/jwks.json
01:38:21.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:38:21.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:38:21.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:38:21.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:38:21.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:38:21.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-2/data/info.pb
01:38:21.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.111.252
01:38:21.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37363
01:38:21.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:38:21.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:38:21.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:38:21.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:38:21.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:38:21.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-2/logs
01:38:21.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:38:21.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:38:21.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:38:21.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:38:21.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:38:21.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:38:21.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:38:21.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 00:02:55 UTC on ccycloud.quasar-qgnmee.root.comops.site
01:38:21.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675483
01:38:21.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:38:21.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:21.230895 22453 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:38:21.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:21.232179 22453 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:38:21.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:21.254033 22476 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:38:21.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:21.978278 22482 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:38:21.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:21.979131 22482 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a0e4466388e452592121c40b5e26931" member_type: VOTER last_known_addr { host: "127.21.111.254" port: 42895 } } peers { permanent_uuid: "0fd65b39ee41409a85c6a728d91f4051" member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } } peers { permanent_uuid: "f01dea69c1d44542bde61d6b1f657868" member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } }
01:38:22.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.007400 22482 leader_election.cc:288] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 9a0e4466388e452592121c40b5e26931 (127.21.111.254:42895), f01dea69c1d44542bde61d6b1f657868 (127.21.111.252:39629)
01:38:22.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:22.026823 22310 leader_election.cc:334] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer f01dea69c1d44542bde61d6b1f657868 (127.21.111.252:39629): Network error: Client connection negotiation failed: client connection to 127.21.111.252:39629: connect: Connection refused (error 111)
01:38:22.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.050582 22426 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0fd65b39ee41409a85c6a728d91f4051" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "9a0e4466388e452592121c40b5e26931" is_pre_election: true
01:38:22.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.051930 22426 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 9a0e4466388e452592121c40b5e26931 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0fd65b39ee41409a85c6a728d91f4051 in term 1.
01:38:22.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.053967 22311 leader_election.cc:302] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 0fd65b39ee41409a85c6a728d91f4051, 9a0e4466388e452592121c40b5e26931; no voters: f01dea69c1d44542bde61d6b1f657868
01:38:22.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.055424 22482 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [term 1 FOLLOWER]: Leader pre-election won for term 2
01:38:22.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.055845 22482 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
01:38:22.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.056267 22482 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [term 1 FOLLOWER]: Advancing to term 2
01:38:22.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.070326 22482 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a0e4466388e452592121c40b5e26931" member_type: VOTER last_known_addr { host: "127.21.111.254" port: 42895 } } peers { permanent_uuid: "0fd65b39ee41409a85c6a728d91f4051" member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } } peers { permanent_uuid: "f01dea69c1d44542bde61d6b1f657868" member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } }
01:38:22.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.074313 22426 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0fd65b39ee41409a85c6a728d91f4051" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "9a0e4466388e452592121c40b5e26931"
01:38:22.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.075070 22426 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 9a0e4466388e452592121c40b5e26931 [term 1 FOLLOWER]: Advancing to term 2
01:38:22.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.088019 22482 leader_election.cc:288] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [CANDIDATE]: Term 2 election: Requested vote from peers 9a0e4466388e452592121c40b5e26931 (127.21.111.254:42895), f01dea69c1d44542bde61d6b1f657868 (127.21.111.252:39629)
01:38:22.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:22.093071 22310 leader_election.cc:334] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer f01dea69c1d44542bde61d6b1f657868 (127.21.111.252:39629): Network error: Client connection negotiation failed: client connection to 127.21.111.252:39629: connect: Connection refused (error 111)
01:38:22.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.096824 22426 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 9a0e4466388e452592121c40b5e26931 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0fd65b39ee41409a85c6a728d91f4051 in term 2.
01:38:22.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.098522 22311 leader_election.cc:302] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 0fd65b39ee41409a85c6a728d91f4051, 9a0e4466388e452592121c40b5e26931; no voters: f01dea69c1d44542bde61d6b1f657868
01:38:22.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.100093 22482 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [term 2 FOLLOWER]: Leader election won for term 2
01:38:22.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.109766 22482 raft_consensus.cc:686] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [term 2 LEADER]: Becoming Leader. State: Replica: 0fd65b39ee41409a85c6a728d91f4051, State: Running, Role: LEADER
01:38:22.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.111275 22482 consensus_queue.cc:237] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a0e4466388e452592121c40b5e26931" member_type: VOTER last_known_addr { host: "127.21.111.254" port: 42895 } } peers { permanent_uuid: "0fd65b39ee41409a85c6a728d91f4051" member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } } peers { permanent_uuid: "f01dea69c1d44542bde61d6b1f657868" member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } }
01:38:22.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.135200 22487 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0fd65b39ee41409a85c6a728d91f4051. Latest consensus state: current_term: 2 leader_uuid: "0fd65b39ee41409a85c6a728d91f4051" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a0e4466388e452592121c40b5e26931" member_type: VOTER last_known_addr { host: "127.21.111.254" port: 42895 } } peers { permanent_uuid: "0fd65b39ee41409a85c6a728d91f4051" member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } } peers { permanent_uuid: "f01dea69c1d44542bde61d6b1f657868" member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } } }
01:38:22.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.136662 22487 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [sys.catalog]: This master's current role is: LEADER
01:38:22.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.142941 22489 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
01:38:22.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.155272 22489 catalog_manager.cc:1484] Initializing Kudu cluster ID...
01:38:22.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.161424 22489 catalog_manager.cc:1259] Loaded cluster ID: ba1632ff75af4af6ac42277a5ebc6ae1
01:38:22.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.161954 22489 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
01:38:22.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.169189 22489 catalog_manager.cc:1504] Loading token signing keys...
01:38:22.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.175076 22489 catalog_manager.cc:5879] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051: Loaded TSK: 0
01:38:22.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.177939 22489 catalog_manager.cc:1514] Initializing in-progress tserver states...
01:38:22.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:22.569806 22310 consensus_peers.cc:487] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 -> Peer f01dea69c1d44542bde61d6b1f657868 (127.21.111.252:39629): Couldn't send request to peer f01dea69c1d44542bde61d6b1f657868. Status: Network error: Client connection negotiation failed: client connection to 127.21.111.252:39629: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
01:38:22.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.625293 22426 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 9a0e4466388e452592121c40b5e26931 [term 2 FOLLOWER]: Refusing update from remote peer 0fd65b39ee41409a85c6a728d91f4051: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
01:38:22.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.628366 22487 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9a0e4466388e452592121c40b5e26931" member_type: VOTER last_known_addr { host: "127.21.111.254" port: 42895 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.001s
01:38:22.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:22.647673 22475 debug-util.cc:398] Leaking SignalData structure 0x7b0800014920 after lost signal to thread 22453
01:38:22.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:22.699617 22475 kernel_stack_watchdog.cc:198] Thread 22453 stuck at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:637 for 391ms:
01:38:22.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
01:38:22.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
01:38:22.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
01:38:22.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
01:38:22.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:21.254027 22477 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:38:22.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.690213 22490 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9a0e4466388e452592121c40b5e26931 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0fd65b39ee41409a85c6a728d91f4051. Latest consensus state: current_term: 2 leader_uuid: "0fd65b39ee41409a85c6a728d91f4051" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a0e4466388e452592121c40b5e26931" member_type: VOTER last_known_addr { host: "127.21.111.254" port: 42895 } } peers { permanent_uuid: "0fd65b39ee41409a85c6a728d91f4051" member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } } peers { permanent_uuid: "f01dea69c1d44542bde61d6b1f657868" member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } } }
01:38:22.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.705437 22490 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9a0e4466388e452592121c40b5e26931 [sys.catalog]: This master's current role is: FOLLOWER
01:38:22.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:22.706257 22453 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.451s user 0.001s sys 0.000s
01:38:22.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:22.706598 22453 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.452s user 0.001s sys 0.000s
01:38:22.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.708130 22487 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "0fd65b39ee41409a85c6a728d91f4051" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a0e4466388e452592121c40b5e26931" member_type: VOTER last_known_addr { host: "127.21.111.254" port: 42895 } } peers { permanent_uuid: "0fd65b39ee41409a85c6a728d91f4051" member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } } peers { permanent_uuid: "f01dea69c1d44542bde61d6b1f657868" member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } } }
01:38:22.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.708839 22487 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [sys.catalog]: This master's current role is: LEADER
01:38:22.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:22.713840 22479 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:38:22.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.714965 22490 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9a0e4466388e452592121c40b5e26931 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "0fd65b39ee41409a85c6a728d91f4051" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a0e4466388e452592121c40b5e26931" member_type: VOTER last_known_addr { host: "127.21.111.254" port: 42895 } } peers { permanent_uuid: "0fd65b39ee41409a85c6a728d91f4051" member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } } peers { permanent_uuid: "f01dea69c1d44542bde61d6b1f657868" member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } } }
01:38:22.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.715613 22490 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9a0e4466388e452592121c40b5e26931 [sys.catalog]: This master's current role is: FOLLOWER
01:38:22.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.719985 22453 server_base.cc:1034] running on GCE node
01:38:22.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.721388 22453 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:38:22.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.732775 22453 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:38:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.734251 22453 hybrid_clock.cc:648] HybridClock initialized: now 1790473102734223 us; error 41 us; skew 500 ppm
01:38:22.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.735314 22453 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:38:22.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 01:38:22 dist-test-slave-37dq krb5kdc[21962](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790473102, etypes {rep=17 tkt=17 ses=17}, kudu/127.21.111.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:38:22.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.742870 22453 init.cc:377] Logged in from keytab as kudu/127.21.111.252@KRBTEST.COM (short username kudu)
01:38:22.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.759604 22453 webserver.cc:460] Webserver started at http://127.21.111.252:37363/ using document root <none> and password file <none>
01:38:22.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.760449 22453 fs_manager.cc:359] Metadata directory not provided
01:38:22.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.760725 22453 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:38:22.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.770503 22453 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.002s sys 0.003s
01:38:22.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.775331 22503 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:38:22.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.776413 22453 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.001s
01:38:22.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.776747 22453 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-2/data,/tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-2/wal
01:38:22.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f01dea69c1d44542bde61d6b1f657868"
01:38:22.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 01:38:13 on dist-test-slave-37dq"
01:38:22.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.778877 22453 fs_report.cc:383] FS layout report
01:38:22.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:38:22.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-2/wal
01:38:22.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-2/wal
01:38:22.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-2/data/data
01:38:22.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:38:22.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:38:22.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:38:22.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:38:22.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:38:22.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:38:22.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:38:22.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:38:22.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:38:22.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:38:22.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:38:22.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.816066 22453 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:38:22.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.822059 22453 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:38:22.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.822502 22453 kserver.cc:163] Server-wide thread pool size limit: 3276
01:38:22.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.913769 22453 rpc_server.cc:307] RPC server started. Bound to: 127.21.111.252:39629
01:38:22.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.914193 22554 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.111.252:39629 every 8 connection(s)
01:38:22.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.917392 22453 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/master-2/data/info.pb
01:38:22.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.921969 21951 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu as pid 22453
01:38:22.922 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:38:22.925 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.21.111.253:37139(127.21.111.253:37139)] connecting to peer
01:38:22.926 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.21.111.254:42895(127.21.111.254:42895)] connecting to peer
01:38:22.929 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.21.111.252:39629(127.21.111.252:39629)] connecting to peer
01:38:22.929 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.21.111.253:37139(127.21.111.253:37139)] Successfully connected to peer
01:38:22.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.933805 22555 sys_catalog.cc:263] Verifying existing consensus state
01:38:22.934 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.21.111.252:39629(127.21.111.252:39629)] Successfully connected to peer
01:38:22.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.941509 22555 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868: Bootstrap starting.
01:38:22.941 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.21.111.254:42895(127.21.111.254:42895)] Successfully connected to peer
01:38:22.951 [DEBUG - kudu-nio-0] (SecurityContext.java:225) Refreshing Kerberos credentials...
01:38:22.951 [DEBUG - kudu-nio-0] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/krb5kdc/krb5cc
01:38:22.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 01:38:22 dist-test-slave-37dq krb5kdc[21962](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790473093, etypes {rep=17 tkt=17 ses=17}, test-user@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:38:22.955 [DEBUG - kudu-nio-0] (SecurityUtil.java:119) Logged in as subject: test-user@KRBTEST.COM
01:38:22.955 [ERROR - kudu-nio-0] (SecurityContext.java:249) Attempted to refresh Kerberos credentials from ticket cache but found that the new Kerberos principal test-user@KRBTEST.COM did not match the original principal test-admin@KRBTEST.COM. Ignoring.
01:38:22.956 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.21.111.253
01:38:22.961 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.21.111.254
01:38:22.983 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.21.111.252
01:38:22.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:22.994362 22555 log.cc:824] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868: Log is configured to *not* fsync() on all Append() calls
01:38:23.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 01:38:23 dist-test-slave-37dq krb5kdc[21962](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790473093, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.21.111.254@KRBTEST.COM
01:38:23.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 01:38:23 dist-test-slave-37dq krb5kdc[21962](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790473098, etypes {rep=17 tkt=17 ses=17}, kudu/127.21.111.253@KRBTEST.COM for kudu/127.21.111.252@KRBTEST.COM
01:38:23.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:23.016592 22555 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
01:38:23.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:23.017503 22555 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868: Bootstrap complete.
01:38:23.025 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.111.253:37139(127.21.111.253:37139)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
01:38:23.025 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.253:37139: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
01:38:23.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:23.028656 22560 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:23.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:22.929821 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:47520 (local address 127.21.111.253:37139)
01:38:23.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:22.948920 (+ 19099us) server_negotiation.cc:206] Beginning negotiation
01:38:23.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:22.948947 (+ 27us) server_negotiation.cc:399] Waiting for connection header
01:38:23.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:22.948993 (+ 46us) server_negotiation.cc:407] Connection header received
01:38:23.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:22.949134 (+ 141us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:23.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:22.949160 (+ 26us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:23.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:22.949456 (+ 296us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:23.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:22.950241 (+ 785us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:23.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:22.980191 (+ 29950us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
01:38:23.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:22.984743 (+ 4552us) server_negotiation.cc:377] Sending TLS_HANDSHAKE NegotiatePB response
01:38:23.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.022166 (+ 37423us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
01:38:23.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.023092 (+ 926us) server_negotiation.cc:377] Sending TLS_HANDSHAKE NegotiatePB response
01:38:23.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.023333 (+ 241us) server_negotiation.cc:662] Negotiated TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 TLSv1.2 Kx=ECDH Au=RSA Enc=AESGCM(128) Mac=AEAD
01:38:23.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.026906 (+ 3573us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:47520: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:47520): Cannot send after transport endpoint shutdown (error 108)
01:38:23.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":17278,"spinlock_wait_cycles":317696,"thread_start_us":18348,"threads_started":1}
01:38:23.038 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xed717038, L:/127.0.0.1:60504 - R:/127.21.111.254:42895] using SASL/GSSAPI
01:38:23.049 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.21.111.253:37139(127.21.111.253:37139)] handling channelInactive
01:38:23.051 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.111.252:39629(127.21.111.252:39629)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
01:38:23.051 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.252:39629: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
01:38:23.052 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629; will retry
01:38:23.052 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.21.111.252:39629(127.21.111.252:39629)] handling channelInactive
01:38:23.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:23.055330 22561 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:23.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:22.935152 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:56036 (local address 127.21.111.252:39629)
01:38:23.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:22.937659 (+ 2507us) server_negotiation.cc:206] Beginning negotiation
01:38:23.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:22.937697 (+ 38us) server_negotiation.cc:399] Waiting for connection header
01:38:23.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:22.937764 (+ 67us) server_negotiation.cc:407] Connection header received
01:38:23.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:22.950837 (+ 13073us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:23.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:22.950888 (+ 51us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:23.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:22.951489 (+ 601us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:23.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:22.951901 (+ 412us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:23.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:22.989444 (+ 37543us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
01:38:23.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:22.992412 (+ 2968us) server_negotiation.cc:377] Sending TLS_HANDSHAKE NegotiatePB response
01:38:23.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.049214 (+ 56802us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
01:38:23.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.050154 (+ 940us) server_negotiation.cc:377] Sending TLS_HANDSHAKE NegotiatePB response
01:38:23.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.050420 (+ 266us) server_negotiation.cc:662] Negotiated TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 TLSv1.2 Kx=ECDH Au=RSA Enc=AESGCM(128) Mac=AEAD
01:38:23.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.053251 (+ 2831us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:56036: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:56036): Cannot send after transport endpoint shutdown (error 108)
01:38:23.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":1914,"thread_start_us":1497,"threads_started":1}
01:38:23.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:23.053421 22555 raft_consensus.cc:348] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a0e4466388e452592121c40b5e26931" member_type: VOTER last_known_addr { host: "127.21.111.254" port: 42895 } } peers { permanent_uuid: "0fd65b39ee41409a85c6a728d91f4051" member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } } peers { permanent_uuid: "f01dea69c1d44542bde61d6b1f657868" member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } }
01:38:23.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:23.058133 22555 raft_consensus.cc:729] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: f01dea69c1d44542bde61d6b1f657868, State: Initialized, Role: FOLLOWER
01:38:23.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:23.059075 22555 consensus_queue.cc:260] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a0e4466388e452592121c40b5e26931" member_type: VOTER last_known_addr { host: "127.21.111.254" port: 42895 } } peers { permanent_uuid: "0fd65b39ee41409a85c6a728d91f4051" member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } } peers { permanent_uuid: "f01dea69c1d44542bde61d6b1f657868" member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } }
01:38:23.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:23.063192 22555 sys_catalog.cc:564] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868 [sys.catalog]: configured and running, proceeding with master startup.
01:38:23.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:23.063825 22530 raft_consensus.cc:3037] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868 [term 1 FOLLOWER]: Advancing to term 2
01:38:23.067 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.253:37139(127.21.111.253:37139)] connecting to peer
01:38:23.068 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.252:39629(127.21.111.252:39629)] connecting to peer
01:38:23.068 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.21.111.253:37139(127.21.111.253:37139)] Successfully connected to peer
01:38:23.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:23.062350 22565 sys_catalog.cc:455] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a0e4466388e452592121c40b5e26931" member_type: VOTER last_known_addr { host: "127.21.111.254" port: 42895 } } peers { permanent_uuid: "0fd65b39ee41409a85c6a728d91f4051" member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } } peers { permanent_uuid: "f01dea69c1d44542bde61d6b1f657868" member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } } }
01:38:23.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:23.069475 22565 sys_catalog.cc:458] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868 [sys.catalog]: This master's current role is: FOLLOWER
01:38:23.068 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.21.111.252:39629(127.21.111.252:39629)] Successfully connected to peer
01:38:23.074 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.111.252:39629(127.21.111.252:39629)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:23.074 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.252:39629: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:23.075 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.21.111.252:39629(127.21.111.252:39629)] handling channelInactive
01:38:23.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:23.076647 22561 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:23.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.071058 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:56048 (local address 127.21.111.252:39629)
01:38:23.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.072942 (+ 1884us) server_negotiation.cc:206] Beginning negotiation
01:38:23.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.072966 (+ 24us) server_negotiation.cc:399] Waiting for connection header
01:38:23.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.073012 (+ 46us) server_negotiation.cc:407] Connection header received
01:38:23.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.073127 (+ 115us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:23.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.073152 (+ 25us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:23.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.073356 (+ 204us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:23.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.073515 (+ 159us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:23.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.075705 (+ 2190us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:56048: BlockingRecv error: recv got EOF from 127.0.0.1:56048 (error 108)
01:38:23.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":1517}
01:38:23.077 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.111.253:37139(127.21.111.253:37139)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:23.078 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.253:37139: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:23.078 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629; will retry
01:38:23.078 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.21.111.253:37139(127.21.111.253:37139)] handling channelInactive
01:38:23.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:23.080451 22560 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:23.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.070594 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:47524 (local address 127.21.111.253:37139)
01:38:23.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.071616 (+ 1022us) server_negotiation.cc:206] Beginning negotiation
01:38:23.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.071641 (+ 25us) server_negotiation.cc:399] Waiting for connection header
01:38:23.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.071684 (+ 43us) server_negotiation.cc:407] Connection header received
01:38:23.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.071811 (+ 127us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:23.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.071837 (+ 26us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:23.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.072060 (+ 223us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:23.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.072229 (+ 169us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:23.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.079463 (+ 7234us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:47524: BlockingRecv error: recv got EOF from 127.0.0.1:47524 (error 108)
01:38:23.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":582}
01:38:23.085 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.253:37139(127.21.111.253:37139)] connecting to peer
01:38:23.086 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.252:39629(127.21.111.252:39629)] connecting to peer
01:38:23.087 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.21.111.253:37139(127.21.111.253:37139)] Successfully connected to peer
01:38:23.088 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.21.111.252:39629(127.21.111.252:39629)] Successfully connected to peer
01:38:23.093 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.111.253:37139(127.21.111.253:37139)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:23.093 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.253:37139: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:23.094 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.21.111.253:37139(127.21.111.253:37139)] handling channelInactive
01:38:23.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:23.095754 22560 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:23.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.091331 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:47538 (local address 127.21.111.253:37139)
01:38:23.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.092452 (+ 1121us) server_negotiation.cc:206] Beginning negotiation
01:38:23.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.092487 (+ 35us) server_negotiation.cc:399] Waiting for connection header
01:38:23.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.092561 (+ 74us) server_negotiation.cc:407] Connection header received
01:38:23.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.092724 (+ 163us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:23.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.092761 (+ 37us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:23.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.093059 (+ 298us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:23.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.093295 (+ 236us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:23.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.094795 (+ 1500us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:47538: BlockingRecv error: recv got EOF from 127.0.0.1:47538 (error 108)
01:38:23.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":619}
01:38:23.097 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.111.252:39629(127.21.111.252:39629)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:23.097 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.252:39629: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:23.097 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629; will retry
01:38:23.098 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.21.111.252:39629(127.21.111.252:39629)] handling channelInactive
01:38:23.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:23.099653 22561 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:23.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.091825 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:56050 (local address 127.21.111.252:39629)
01:38:23.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.094243 (+ 2418us) server_negotiation.cc:206] Beginning negotiation
01:38:23.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.094278 (+ 35us) server_negotiation.cc:399] Waiting for connection header
01:38:23.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.094329 (+ 51us) server_negotiation.cc:407] Connection header received
01:38:23.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.094494 (+ 165us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:23.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.094530 (+ 36us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:23.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.094809 (+ 279us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:23.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.095007 (+ 198us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:23.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.098670 (+ 3663us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:56050: BlockingRecv error: recv got EOF from 127.0.0.1:56050 (error 108)
01:38:23.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":1858}
01:38:23.106 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.253:37139(127.21.111.253:37139)] connecting to peer
01:38:23.106 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.252:39629(127.21.111.252:39629)] connecting to peer
01:38:23.107 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.21.111.253:37139(127.21.111.253:37139)] Successfully connected to peer
01:38:23.115 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.111.253:37139(127.21.111.253:37139)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:23.115 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.253:37139: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:23.115 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.21.111.253:37139(127.21.111.253:37139)] handling channelInactive
01:38:23.116 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.21.111.252:39629(127.21.111.252:39629)] Successfully connected to peer
01:38:23.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:23.117328 22560 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:23.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.113709 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:47548 (local address 127.21.111.253:37139)
01:38:23.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.114178 (+ 469us) server_negotiation.cc:206] Beginning negotiation
01:38:23.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.114203 (+ 25us) server_negotiation.cc:399] Waiting for connection header
01:38:23.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.114247 (+ 44us) server_negotiation.cc:407] Connection header received
01:38:23.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.114369 (+ 122us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:23.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.114394 (+ 25us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:23.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.114594 (+ 200us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:23.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.114760 (+ 166us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:23.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.116401 (+ 1641us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:47548: BlockingRecv error: recv got EOF from 127.0.0.1:47548 (error 108)
01:38:23.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":82}
01:38:23.121 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.111.252:39629(127.21.111.252:39629)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:23.121 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.252:39629: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:23.122 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629; will retry
01:38:23.122 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.21.111.252:39629(127.21.111.252:39629)] handling channelInactive
01:38:23.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:23.124387 22561 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:23.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.118049 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:56056 (local address 127.21.111.252:39629)
01:38:23.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.119359 (+ 1310us) server_negotiation.cc:206] Beginning negotiation
01:38:23.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.119382 (+ 23us) server_negotiation.cc:399] Waiting for connection header
01:38:23.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.119422 (+ 40us) server_negotiation.cc:407] Connection header received
01:38:23.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.119541 (+ 119us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:23.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.119565 (+ 24us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:23.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.119753 (+ 188us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:23.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.119918 (+ 165us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:23.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.123452 (+ 3534us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:56056: BlockingRecv error: recv got EOF from 127.0.0.1:56056 (error 108)
01:38:23.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":929}
01:38:23.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:23.120761 22565 sys_catalog.cc:455] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0fd65b39ee41409a85c6a728d91f4051. Latest consensus state: current_term: 2 leader_uuid: "0fd65b39ee41409a85c6a728d91f4051" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a0e4466388e452592121c40b5e26931" member_type: VOTER last_known_addr { host: "127.21.111.254" port: 42895 } } peers { permanent_uuid: "0fd65b39ee41409a85c6a728d91f4051" member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } } peers { permanent_uuid: "f01dea69c1d44542bde61d6b1f657868" member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } } }
01:38:23.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:23.126628 22565 sys_catalog.cc:458] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868 [sys.catalog]: This master's current role is: FOLLOWER
01:38:23.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:23.129146 22570 sys_catalog.cc:455] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "0fd65b39ee41409a85c6a728d91f4051" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a0e4466388e452592121c40b5e26931" member_type: VOTER last_known_addr { host: "127.21.111.254" port: 42895 } } peers { permanent_uuid: "0fd65b39ee41409a85c6a728d91f4051" member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } } peers { permanent_uuid: "f01dea69c1d44542bde61d6b1f657868" member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } } }
01:38:23.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:23.130164 22570 sys_catalog.cc:458] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868 [sys.catalog]: This master's current role is: FOLLOWER
01:38:23.145 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.253:37139(127.21.111.253:37139)] connecting to peer
01:38:23.146 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.252:39629(127.21.111.252:39629)] connecting to peer
01:38:23.146 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.21.111.252:39629(127.21.111.252:39629)] Successfully connected to peer
01:38:23.146 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.21.111.253:37139(127.21.111.253:37139)] Successfully connected to peer
01:38:23.149 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.111.253:37139(127.21.111.253:37139)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:23.149 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.253:37139: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:23.150 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.21.111.253:37139(127.21.111.253:37139)] handling channelInactive
01:38:23.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:23.151705 22560 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:23.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.147541 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:47556 (local address 127.21.111.253:37139)
01:38:23.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.148049 (+ 508us) server_negotiation.cc:206] Beginning negotiation
01:38:23.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.148074 (+ 25us) server_negotiation.cc:399] Waiting for connection header
01:38:23.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.148116 (+ 42us) server_negotiation.cc:407] Connection header received
01:38:23.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.148500 (+ 384us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:23.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.148547 (+ 47us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:23.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.148773 (+ 226us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:23.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.148956 (+ 183us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:23.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.150759 (+ 1803us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:47556: BlockingRecv error: recv got EOF from 127.0.0.1:47556 (error 108)
01:38:23.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":113}
01:38:23.155 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.111.252:39629(127.21.111.252:39629)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:23.157 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.252:39629: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:23.157 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629; will retry
01:38:23.157 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.21.111.252:39629(127.21.111.252:39629)] handling channelInactive
01:38:23.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:23.159703 22561 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:23.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.148791 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:56058 (local address 127.21.111.252:39629)
01:38:23.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.149322 (+ 531us) server_negotiation.cc:206] Beginning negotiation
01:38:23.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.149345 (+ 23us) server_negotiation.cc:399] Waiting for connection header
01:38:23.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.149384 (+ 39us) server_negotiation.cc:407] Connection header received
01:38:23.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.153648 (+ 4264us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:23.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.153685 (+ 37us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:23.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.153934 (+ 249us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:23.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.154106 (+ 172us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:23.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.158437 (+ 4331us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:56058: BlockingRecv error: recv got EOF from 127.0.0.1:56058 (error 108)
01:38:23.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":142}
01:38:23.186 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.253:37139(127.21.111.253:37139)] connecting to peer
01:38:23.187 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.252:39629(127.21.111.252:39629)] connecting to peer
01:38:23.188 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.21.111.253:37139(127.21.111.253:37139)] Successfully connected to peer
01:38:23.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 01:38:23 dist-test-slave-37dq krb5kdc[21962](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790473102, etypes {rep=17 tkt=17 ses=17}, kudu/127.21.111.252@KRBTEST.COM for kudu/127.21.111.254@KRBTEST.COM
01:38:23.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:23.190627 22579 catalog_manager.cc:1259] Loaded cluster ID: ba1632ff75af4af6ac42277a5ebc6ae1
01:38:23.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:23.190866 22579 catalog_manager.cc:1552] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868: loading cluster ID for follower catalog manager: success
01:38:23.196 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.21.111.252:39629(127.21.111.252:39629)] Successfully connected to peer
01:38:23.198 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.111.252:39629(127.21.111.252:39629)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:23.198 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.252:39629: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:23.198 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.21.111.252:39629(127.21.111.252:39629)] handling channelInactive
01:38:23.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:23.200096 22561 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:23.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.192839 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:56064 (local address 127.21.111.252:39629)
01:38:23.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.193321 (+ 482us) server_negotiation.cc:206] Beginning negotiation
01:38:23.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.193347 (+ 26us) server_negotiation.cc:399] Waiting for connection header
01:38:23.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.196585 (+ 3238us) server_negotiation.cc:407] Connection header received
01:38:23.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.196739 (+ 154us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:23.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.196774 (+ 35us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:23.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.197004 (+ 230us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:23.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.197149 (+ 145us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:23.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.199159 (+ 2010us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:56064: BlockingRecv error: recv got EOF from 127.0.0.1:56064 (error 108)
01:38:23.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":82}
01:38:23.201 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.111.253:37139(127.21.111.253:37139)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:23.201 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.253:37139: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:23.201 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629; will retry
01:38:23.202 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.21.111.253:37139(127.21.111.253:37139)] handling channelInactive
01:38:23.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:23.201076 22579 catalog_manager.cc:1574] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868: acquiring CA information for follower catalog manager: success
01:38:23.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:23.203552 22560 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:23.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.194396 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:47568 (local address 127.21.111.253:37139)
01:38:23.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.194869 (+ 473us) server_negotiation.cc:206] Beginning negotiation
01:38:23.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.194894 (+ 25us) server_negotiation.cc:399] Waiting for connection header
01:38:23.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.194938 (+ 44us) server_negotiation.cc:407] Connection header received
01:38:23.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.195096 (+ 158us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:23.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.195121 (+ 25us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:23.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.195346 (+ 225us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:23.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.195529 (+ 183us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:23.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.202643 (+ 7114us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:47568: BlockingRecv error: recv got EOF from 127.0.0.1:47568 (error 108)
01:38:23.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":114}
01:38:23.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:23.209816 22579 catalog_manager.cc:1602] T 00000000000000000000000000000000 P f01dea69c1d44542bde61d6b1f657868: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
01:38:23.226 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.253:37139(127.21.111.253:37139)] connecting to peer
01:38:23.226 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.252:39629(127.21.111.252:39629)] connecting to peer
01:38:23.227 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.21.111.252:39629(127.21.111.252:39629)] Successfully connected to peer
01:38:23.228 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.21.111.253:37139(127.21.111.253:37139)] Successfully connected to peer
01:38:23.230 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.111.252:39629(127.21.111.252:39629)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:23.231 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.252:39629: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:23.231 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.21.111.252:39629(127.21.111.252:39629)] handling channelInactive
01:38:23.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:23.233081 22561 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:23.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.228999 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:56078 (local address 127.21.111.252:39629)
01:38:23.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.229482 (+ 483us) server_negotiation.cc:206] Beginning negotiation
01:38:23.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.229507 (+ 25us) server_negotiation.cc:399] Waiting for connection header
01:38:23.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.229549 (+ 42us) server_negotiation.cc:407] Connection header received
01:38:23.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.229661 (+ 112us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:23.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.229687 (+ 26us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:23.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.230137 (+ 450us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:23.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.230321 (+ 184us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:23.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.232123 (+ 1802us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:56078: BlockingRecv error: recv got EOF from 127.0.0.1:56078 (error 108)
01:38:23.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":100}
01:38:23.236 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.111.253:37139(127.21.111.253:37139)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:23.236 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.253:37139: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:23.236 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629; will retry
01:38:23.237 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.21.111.253:37139(127.21.111.253:37139)] handling channelInactive
01:38:23.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:23.242779 22582 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:23.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.229999 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:47576 (local address 127.21.111.253:37139)
01:38:23.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.235216 (+ 5217us) server_negotiation.cc:206] Beginning negotiation
01:38:23.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.235235 (+ 19us) server_negotiation.cc:399] Waiting for connection header
01:38:23.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.235279 (+ 44us) server_negotiation.cc:407] Connection header received
01:38:23.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.235416 (+ 137us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:23.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.235441 (+ 25us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:23.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.235644 (+ 203us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:23.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.240907 (+ 5263us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:23.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.241640 (+ 733us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:47576: BlockingRecv error: recv got EOF from 127.0.0.1:47576 (error 108)
01:38:23.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":4820,"thread_start_us":4341,"threads_started":1}
01:38:23.345 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.253:37139(127.21.111.253:37139)] connecting to peer
01:38:23.346 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.252:39629(127.21.111.252:39629)] connecting to peer
01:38:23.347 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.21.111.253:37139(127.21.111.253:37139)] Successfully connected to peer
01:38:23.353 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.111.253:37139(127.21.111.253:37139)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:23.354 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.253:37139: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:23.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:23.357142 22560 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:23.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.351551 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:47592 (local address 127.21.111.253:37139)
01:38:23.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.352056 (+ 505us) server_negotiation.cc:206] Beginning negotiation
01:38:23.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.352082 (+ 26us) server_negotiation.cc:399] Waiting for connection header
01:38:23.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.352133 (+ 51us) server_negotiation.cc:407] Connection header received
01:38:23.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.352286 (+ 153us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:23.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.352311 (+ 25us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:23.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.352578 (+ 267us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:23.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.352759 (+ 181us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:23.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.356179 (+ 3420us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:47592: BlockingRecv error: recv got EOF from 127.0.0.1:47592 (error 108)
01:38:23.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":94}
01:38:23.358 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.21.111.252:39629(127.21.111.252:39629)] Successfully connected to peer
01:38:23.360 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.21.111.253:37139(127.21.111.253:37139)] handling channelInactive
01:38:23.360 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.111.252:39629(127.21.111.252:39629)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:23.361 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.252:39629: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:23.361 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629; will retry
01:38:23.362 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.21.111.252:39629(127.21.111.252:39629)] handling channelInactive
01:38:23.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:23.362840 22561 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:23.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.359010 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:56090 (local address 127.21.111.252:39629)
01:38:23.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.359520 (+ 510us) server_negotiation.cc:206] Beginning negotiation
01:38:23.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.359549 (+ 29us) server_negotiation.cc:399] Waiting for connection header
01:38:23.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.359591 (+ 42us) server_negotiation.cc:407] Connection header received
01:38:23.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.359718 (+ 127us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:23.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.359744 (+ 26us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:23.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.359967 (+ 223us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:23.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.360148 (+ 181us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:23.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.361930 (+ 1782us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:56090: BlockingRecv error: recv got EOF from 127.0.0.1:56090 (error 108)
01:38:23.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":129}
01:38:23.465 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.253:37139(127.21.111.253:37139)] connecting to peer
01:38:23.466 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.252:39629(127.21.111.252:39629)] connecting to peer
01:38:23.467 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.21.111.253:37139(127.21.111.253:37139)] Successfully connected to peer
01:38:23.469 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.21.111.252:39629(127.21.111.252:39629)] Successfully connected to peer
01:38:23.476 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.111.252:39629(127.21.111.252:39629)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:23.477 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.252:39629: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:23.477 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.21.111.252:39629(127.21.111.252:39629)] handling channelInactive
01:38:23.478 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.111.253:37139(127.21.111.253:37139)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:23.479 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.253:37139: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:23.479 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629; will retry
01:38:23.480 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.21.111.253:37139(127.21.111.253:37139)] handling channelInactive
01:38:23.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:23.481784 22560 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:23.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.473140 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:47608 (local address 127.21.111.253:37139)
01:38:23.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.473626 (+ 486us) server_negotiation.cc:206] Beginning negotiation
01:38:23.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.473669 (+ 43us) server_negotiation.cc:399] Waiting for connection header
01:38:23.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.473710 (+ 41us) server_negotiation.cc:407] Connection header received
01:38:23.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.473844 (+ 134us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:23.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.473872 (+ 28us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:23.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.474076 (+ 204us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:23.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.474230 (+ 154us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:23.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.480797 (+ 6567us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:47608: BlockingRecv error: recv got EOF from 127.0.0.1:47608 (error 108)
01:38:23.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":109}
01:38:23.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:23.483999 22561 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:23.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.472807 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:56104 (local address 127.21.111.252:39629)
01:38:23.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.475215 (+ 2408us) server_negotiation.cc:206] Beginning negotiation
01:38:23.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.475240 (+ 25us) server_negotiation.cc:399] Waiting for connection header
01:38:23.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.475280 (+ 40us) server_negotiation.cc:407] Connection header received
01:38:23.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.475436 (+ 156us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:23.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.475463 (+ 27us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:23.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.475657 (+ 194us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:23.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.475813 (+ 156us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:23.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.483021 (+ 7208us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:56104: BlockingRecv error: recv got EOF from 127.0.0.1:56104 (error 108)
01:38:23.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":2031}
01:38:23.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:23.500448 22487 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "0fd65b39ee41409a85c6a728d91f4051" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a0e4466388e452592121c40b5e26931" member_type: VOTER last_known_addr { host: "127.21.111.254" port: 42895 } } peers { permanent_uuid: "0fd65b39ee41409a85c6a728d91f4051" member_type: VOTER last_known_addr { host: "127.21.111.253" port: 37139 } } peers { permanent_uuid: "f01dea69c1d44542bde61d6b1f657868" member_type: VOTER last_known_addr { host: "127.21.111.252" port: 39629 } } }
01:38:23.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:23.501194 22487 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0fd65b39ee41409a85c6a728d91f4051 [sys.catalog]: This master's current role is: LEADER
01:38:23.965 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.253:37139(127.21.111.253:37139)] connecting to peer
01:38:23.966 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.252:39629(127.21.111.252:39629)] connecting to peer
01:38:23.967 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.21.111.253:37139(127.21.111.253:37139)] Successfully connected to peer
01:38:23.968 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.21.111.252:39629(127.21.111.252:39629)] Successfully connected to peer
01:38:23.974 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.111.252:39629(127.21.111.252:39629)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:23.974 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.252:39629: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:23.975 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.21.111.252:39629(127.21.111.252:39629)] handling channelInactive
01:38:23.975 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.111.253:37139(127.21.111.253:37139)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:23.975 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.253:37139: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:23.976 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629; will retry
01:38:23.976 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.21.111.253:37139(127.21.111.253:37139)] handling channelInactive
01:38:23.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:23.976457 22561 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:23.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.969182 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:56114 (local address 127.21.111.252:39629)
01:38:23.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.969665 (+ 483us) server_negotiation.cc:206] Beginning negotiation
01:38:23.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.969686 (+ 21us) server_negotiation.cc:399] Waiting for connection header
01:38:23.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.969730 (+ 44us) server_negotiation.cc:407] Connection header received
01:38:23.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.969844 (+ 114us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:23.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.969870 (+ 26us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:23.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.970100 (+ 230us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:23.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.970270 (+ 170us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:23.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.975482 (+ 5212us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:56114: BlockingRecv error: recv got EOF from 127.0.0.1:56114 (error 108)
01:38:23.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":111}
01:38:23.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:23.977993 22560 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:23.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.973858 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:47636 (local address 127.21.111.253:37139)
01:38:23.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.974349 (+ 491us) server_negotiation.cc:206] Beginning negotiation
01:38:23.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.974377 (+ 28us) server_negotiation.cc:399] Waiting for connection header
01:38:23.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.974420 (+ 43us) server_negotiation.cc:407] Connection header received
01:38:23.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.974543 (+ 123us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:23.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.974569 (+ 26us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:23.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.974788 (+ 219us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:23.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.974957 (+ 169us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:23.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:23.977027 (+ 2070us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:47636: BlockingRecv error: recv got EOF from 127.0.0.1:47636 (error 108)
01:38:23.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":101}
01:38:24.166 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.253:37139(127.21.111.253:37139)] connecting to peer
01:38:24.166 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.252:39629(127.21.111.252:39629)] connecting to peer
01:38:24.167 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.21.111.252:39629(127.21.111.252:39629)] Successfully connected to peer
01:38:24.167 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.21.111.253:37139(127.21.111.253:37139)] Successfully connected to peer
01:38:24.171 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.111.252:39629(127.21.111.252:39629)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:24.171 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.111.253:37139(127.21.111.253:37139)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:24.171 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.252:39629: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:24.171 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.253:37139: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:24.171 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.21.111.252:39629(127.21.111.252:39629)] handling channelInactive
01:38:24.171 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.21.111.253:37139(127.21.111.253:37139)] handling channelInactive
01:38:24.172 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629; will retry
01:38:24.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:24.172894 22560 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:24.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:24.169327 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:47642 (local address 127.21.111.253:37139)
01:38:24.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:24.169858 (+ 531us) server_negotiation.cc:206] Beginning negotiation
01:38:24.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:24.169881 (+ 23us) server_negotiation.cc:399] Waiting for connection header
01:38:24.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:24.169924 (+ 43us) server_negotiation.cc:407] Connection header received
01:38:24.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:24.170401 (+ 477us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:24.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:24.170433 (+ 32us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:24.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:24.170663 (+ 230us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:24.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:24.170883 (+ 220us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:24.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:24.172021 (+ 1138us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:47642: BlockingRecv error: recv got EOF from 127.0.0.1:47642 (error 108)
01:38:24.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":133}
01:38:24.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:24.173300 22561 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:24.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:24.168583 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:56118 (local address 127.21.111.252:39629)
01:38:24.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:24.169641 (+ 1058us) server_negotiation.cc:206] Beginning negotiation
01:38:24.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:24.169668 (+ 27us) server_negotiation.cc:399] Waiting for connection header
01:38:24.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:24.169714 (+ 46us) server_negotiation.cc:407] Connection header received
01:38:24.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:24.169847 (+ 133us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:24.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:24.169872 (+ 25us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:24.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:24.170087 (+ 215us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:24.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:24.170290 (+ 203us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:24.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:24.172272 (+ 1982us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:56118: BlockingRecv error: recv got EOF from 127.0.0.1:56118 (error 108)
01:38:24.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":620}
01:38:25.606 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.253:37139(127.21.111.253:37139)] connecting to peer
01:38:25.607 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.252:39629(127.21.111.252:39629)] connecting to peer
01:38:25.607 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.21.111.253:37139(127.21.111.253:37139)] Successfully connected to peer
01:38:25.607 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.21.111.252:39629(127.21.111.252:39629)] Successfully connected to peer
01:38:25.610 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.111.253:37139(127.21.111.253:37139)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:25.610 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.253:37139: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:25.610 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.21.111.253:37139(127.21.111.253:37139)] handling channelInactive
01:38:25.610 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.111.252:39629(127.21.111.252:39629)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:25.611 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.252:39629: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:25.611 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.21.111.252:39629(127.21.111.252:39629)] handling channelInactive
01:38:25.612 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629; will retry
01:38:25.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:25.612128 22591 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:25.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:25.607433 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:47656 (local address 127.21.111.253:37139)
01:38:25.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:25.609111 (+ 1678us) server_negotiation.cc:206] Beginning negotiation
01:38:25.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:25.609140 (+ 29us) server_negotiation.cc:399] Waiting for connection header
01:38:25.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:25.609193 (+ 53us) server_negotiation.cc:407] Connection header received
01:38:25.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:25.609341 (+ 148us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:25.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:25.609366 (+ 25us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:25.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:25.609585 (+ 219us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:25.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:25.609774 (+ 189us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:25.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:25.611066 (+ 1292us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:47656: BlockingRecv error: recv got EOF from 127.0.0.1:47656 (error 108)
01:38:25.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":1230,"thread_start_us":955,"threads_started":1}
01:38:25.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:25.613307 22592 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:25.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:25.608490 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:56122 (local address 127.21.111.252:39629)
01:38:25.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:25.609761 (+ 1271us) server_negotiation.cc:206] Beginning negotiation
01:38:25.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:25.609780 (+ 19us) server_negotiation.cc:399] Waiting for connection header
01:38:25.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:25.609822 (+ 42us) server_negotiation.cc:407] Connection header received
01:38:25.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:25.609949 (+ 127us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:25.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:25.609972 (+ 23us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:25.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:25.610184 (+ 212us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:25.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:25.611589 (+ 1405us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:25.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:25.612202 (+ 613us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:56122: BlockingRecv error: recv got EOF from 127.0.0.1:56122 (error 108)
01:38:25.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":936,"thread_start_us":670,"threads_started":1}
01:38:26.225 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.253:37139(127.21.111.253:37139)] connecting to peer
01:38:26.226 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.252:39629(127.21.111.252:39629)] connecting to peer
01:38:26.227 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.21.111.253:37139(127.21.111.253:37139)] Successfully connected to peer
01:38:26.227 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.21.111.252:39629(127.21.111.252:39629)] Successfully connected to peer
01:38:26.230 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.111.252:39629(127.21.111.252:39629)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:26.230 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.252:39629: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:26.230 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.111.253:37139(127.21.111.253:37139)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:26.230 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.253:37139: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:26.230 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.21.111.252:39629(127.21.111.252:39629)] handling channelInactive
01:38:26.231 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629; will retry
01:38:26.231 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.21.111.253:37139(127.21.111.253:37139)] handling channelInactive
01:38:26.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:26.231616 22592 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:26.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:26.228179 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:56140 (local address 127.21.111.252:39629)
01:38:26.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:26.228723 (+ 544us) server_negotiation.cc:206] Beginning negotiation
01:38:26.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:26.228750 (+ 27us) server_negotiation.cc:399] Waiting for connection header
01:38:26.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:26.228799 (+ 49us) server_negotiation.cc:407] Connection header received
01:38:26.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:26.228945 (+ 146us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:26.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:26.228968 (+ 23us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:26.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:26.229184 (+ 216us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:26.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:26.230298 (+ 1114us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:26.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:26.230925 (+ 627us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:56140: BlockingRecv error: recv got EOF from 127.0.0.1:56140 (error 108)
01:38:26.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":99}
01:38:26.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:26.232722 22595 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:26.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:26.228069 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:47664 (local address 127.21.111.253:37139)
01:38:26.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:26.229316 (+ 1247us) server_negotiation.cc:206] Beginning negotiation
01:38:26.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:26.229356 (+ 40us) server_negotiation.cc:399] Waiting for connection header
01:38:26.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:26.229399 (+ 43us) server_negotiation.cc:407] Connection header received
01:38:26.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:26.229532 (+ 133us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:26.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:26.229555 (+ 23us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:26.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:26.229761 (+ 206us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:26.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:26.229942 (+ 181us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:26.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:26.231811 (+ 1869us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:47664: BlockingRecv error: recv got EOF from 127.0.0.1:47664 (error 108)
01:38:26.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":854,"thread_start_us":571,"threads_started":1}
01:38:28.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 01:38:28 dist-test-slave-37dq krb5kdc[21962](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790473108, etypes {rep=17 tkt=17 ses=17}, kudu/127.21.111.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:38:28.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:28.317106 22303 init.cc:295] Successfully reacquired a new kerberos TGT
01:38:29.925 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.253:37139(127.21.111.253:37139)] connecting to peer
01:38:29.926 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.252:39629(127.21.111.252:39629)] connecting to peer
01:38:29.927 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.21.111.253:37139(127.21.111.253:37139)] Successfully connected to peer
01:38:29.928 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.21.111.252:39629(127.21.111.252:39629)] Successfully connected to peer
01:38:29.931 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.111.252:39629(127.21.111.252:39629)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:29.931 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.252:39629: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:29.931 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.21.111.252:39629(127.21.111.252:39629)] handling channelInactive
01:38:29.932 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.111.253:37139(127.21.111.253:37139)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:29.932 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.253:37139: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:29.932 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629; will retry
01:38:29.933 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.21.111.253:37139(127.21.111.253:37139)] handling channelInactive
01:38:29.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:29.932844 22601 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:29.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:29.927683 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:56150 (local address 127.21.111.252:39629)
01:38:29.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:29.929767 (+ 2084us) server_negotiation.cc:206] Beginning negotiation
01:38:29.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:29.929789 (+ 22us) server_negotiation.cc:399] Waiting for connection header
01:38:29.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:29.929839 (+ 50us) server_negotiation.cc:407] Connection header received
01:38:29.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:29.929980 (+ 141us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:29.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:29.930007 (+ 27us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:29.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:29.930225 (+ 218us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:29.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:29.931144 (+ 919us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:29.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:29.931887 (+ 743us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:56150: BlockingRecv error: recv got EOF from 127.0.0.1:56150 (error 108)
01:38:29.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":1604,"thread_start_us":1273,"threads_started":1}
01:38:29.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:29.934827 22602 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:29.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:29.928411 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:47678 (local address 127.21.111.253:37139)
01:38:29.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:29.930993 (+ 2582us) server_negotiation.cc:206] Beginning negotiation
01:38:29.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:29.931028 (+ 35us) server_negotiation.cc:399] Waiting for connection header
01:38:29.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:29.931095 (+ 67us) server_negotiation.cc:407] Connection header received
01:38:29.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:29.931297 (+ 202us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:29.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:29.931321 (+ 24us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:29.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:29.931618 (+ 297us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:29.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:29.931870 (+ 252us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:29.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:29.933631 (+ 1761us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:47678: BlockingRecv error: recv got EOF from 127.0.0.1:47678 (error 108)
01:38:29.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":2065,"thread_start_us":1789,"threads_started":1}
01:38:30.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:30.193115 22447 debug-util.cc:398] Leaking SignalData structure 0x7b080009a000 after lost signal to thread 22377
01:38:30.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:30.193840 22447 debug-util.cc:398] Leaking SignalData structure 0x7b08000ac540 after lost signal to thread 22450
01:38:30.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 01:38:30 dist-test-slave-37dq krb5kdc[21962](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790473110, etypes {rep=17 tkt=17 ses=17}, kudu/127.21.111.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:38:30.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:30.438616 22395 init.cc:295] Successfully reacquired a new kerberos TGT
01:38:30.825 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.253:37139(127.21.111.253:37139)] connecting to peer
01:38:30.826 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.252:39629(127.21.111.252:39629)] connecting to peer
01:38:30.826 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.21.111.252:39629(127.21.111.252:39629)] Successfully connected to peer
01:38:30.826 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.21.111.253:37139(127.21.111.253:37139)] Successfully connected to peer
01:38:30.834 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.111.252:39629(127.21.111.252:39629)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:30.835 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.252:39629: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:30.835 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.21.111.252:39629(127.21.111.252:39629)] handling channelInactive
01:38:30.836 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.111.253:37139(127.21.111.253:37139)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:30.836 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.253:37139: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:30.836 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629; will retry
01:38:30.837 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.21.111.253:37139(127.21.111.253:37139)] handling channelInactive
01:38:30.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:30.837541 22604 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:30.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:30.827997 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:51446 (local address 127.21.111.252:39629)
01:38:30.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:30.833557 (+ 5560us) server_negotiation.cc:206] Beginning negotiation
01:38:30.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:30.833603 (+ 46us) server_negotiation.cc:399] Waiting for connection header
01:38:30.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:30.833673 (+ 70us) server_negotiation.cc:407] Connection header received
01:38:30.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:30.833834 (+ 161us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:30.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:30.833859 (+ 25us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:30.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:30.834107 (+ 248us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:30.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:30.834396 (+ 289us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:30.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:30.836122 (+ 1726us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:51446: BlockingRecv error: recv got EOF from 127.0.0.1:51446 (error 108)
01:38:30.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":4888,"thread_start_us":4626,"threads_started":1}
01:38:30.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:30.839047 22605 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:30.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:30.828692 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:36606 (local address 127.21.111.253:37139)
01:38:30.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:30.834472 (+ 5780us) server_negotiation.cc:206] Beginning negotiation
01:38:30.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:30.834503 (+ 31us) server_negotiation.cc:399] Waiting for connection header
01:38:30.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:30.834553 (+ 50us) server_negotiation.cc:407] Connection header received
01:38:30.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:30.834688 (+ 135us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:30.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:30.834716 (+ 28us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:30.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:30.835022 (+ 306us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:30.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:30.835503 (+ 481us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:30.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:30.837904 (+ 2401us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:36606: BlockingRecv error: recv got EOF from 127.0.0.1:36606 (error 108)
01:38:30.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":5231,"thread_start_us":4984,"threads_started":1}
01:38:32.165 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.253:37139(127.21.111.253:37139)] connecting to peer
01:38:32.166 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.252:39629(127.21.111.252:39629)] connecting to peer
01:38:32.167 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.21.111.253:37139(127.21.111.253:37139)] Successfully connected to peer
01:38:32.167 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.21.111.252:39629(127.21.111.252:39629)] Successfully connected to peer
01:38:32.172 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.111.252:39629(127.21.111.252:39629)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:32.172 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.252:39629: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:32.173 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.111.253:37139(127.21.111.253:37139)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:32.173 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.21.111.252:39629(127.21.111.252:39629)] handling channelInactive
01:38:32.173 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.253:37139: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:32.173 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629; will retry
01:38:32.174 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.21.111.253:37139(127.21.111.253:37139)] handling channelInactive
01:38:32.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:32.174870 22608 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:32.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:32.168402 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:51454 (local address 127.21.111.252:39629)
01:38:32.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:32.171282 (+ 2880us) server_negotiation.cc:206] Beginning negotiation
01:38:32.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:32.171313 (+ 31us) server_negotiation.cc:399] Waiting for connection header
01:38:32.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:32.171384 (+ 71us) server_negotiation.cc:407] Connection header received
01:38:32.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:32.171589 (+ 205us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:32.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:32.171615 (+ 26us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:32.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:32.171991 (+ 376us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:32.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:32.172202 (+ 211us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:32.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:32.173624 (+ 1422us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:51454: BlockingRecv error: recv got EOF from 127.0.0.1:51454 (error 108)
01:38:32.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":2267,"thread_start_us":2340,"threads_started":1}
01:38:32.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:32.175542 22609 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:32.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:32.168773 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:36608 (local address 127.21.111.253:37139)
01:38:32.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:32.171974 (+ 3201us) server_negotiation.cc:206] Beginning negotiation
01:38:32.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:32.172001 (+ 27us) server_negotiation.cc:399] Waiting for connection header
01:38:32.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:32.172044 (+ 43us) server_negotiation.cc:407] Connection header received
01:38:32.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:32.172186 (+ 142us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:32.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:32.172214 (+ 28us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:32.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:32.172445 (+ 231us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:32.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:32.172847 (+ 402us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:32.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:32.174499 (+ 1652us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:36608: BlockingRecv error: recv got EOF from 127.0.0.1:36608 (error 108)
01:38:32.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":2764,"thread_start_us":2487,"threads_started":1}
01:38:32.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 01:38:32 dist-test-slave-37dq krb5kdc[21962](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790473112, etypes {rep=17 tkt=17 ses=17}, kudu/127.21.111.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:38:32.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:32.751513 22499 init.cc:295] Successfully reacquired a new kerberos TGT
01:38:32.786 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.253:37139(127.21.111.253:37139)] connecting to peer
01:38:32.787 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.252:39629(127.21.111.252:39629)] connecting to peer
01:38:32.787 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.21.111.252:39629(127.21.111.252:39629)] Successfully connected to peer
01:38:32.787 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.21.111.253:37139(127.21.111.253:37139)] Successfully connected to peer
01:38:32.790 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.111.252:39629(127.21.111.252:39629)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:32.791 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.252:39629: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:32.791 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.21.111.252:39629(127.21.111.252:39629)] handling channelInactive
01:38:32.792 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.111.253:37139(127.21.111.253:37139)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:32.792 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.253:37139: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:32.793 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629; will retry
01:38:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:32.793071 22611 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:32.788120 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:51468 (local address 127.21.111.252:39629)
01:38:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:32.789342 (+ 1222us) server_negotiation.cc:206] Beginning negotiation
01:38:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:32.789363 (+ 21us) server_negotiation.cc:399] Waiting for connection header
01:38:32.793 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.21.111.253:37139(127.21.111.253:37139)] handling channelInactive
01:38:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:32.789412 (+ 49us) server_negotiation.cc:407] Connection header received
01:38:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:32.789544 (+ 132us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:32.789566 (+ 22us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:32.789797 (+ 231us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:32.791261 (+ 1464us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:32.791944 (+ 683us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:51468: BlockingRecv error: recv got EOF from 127.0.0.1:51468 (error 108)
01:38:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":823,"thread_start_us":515,"threads_started":1}
01:38:32.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:32.794922 22612 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:32.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:32.789139 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:36614 (local address 127.21.111.253:37139)
01:38:32.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:32.791614 (+ 2475us) server_negotiation.cc:206] Beginning negotiation
01:38:32.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:32.791642 (+ 28us) server_negotiation.cc:399] Waiting for connection header
01:38:32.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:32.791687 (+ 45us) server_negotiation.cc:407] Connection header received
01:38:32.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:32.791821 (+ 134us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:32.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:32.791857 (+ 36us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:32.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:32.792069 (+ 212us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:32.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:32.792248 (+ 179us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:32.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:32.793967 (+ 1719us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:36614: BlockingRecv error: recv got EOF from 127.0.0.1:36614 (error 108)
01:38:32.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":2092,"thread_start_us":1856,"threads_started":1}
01:38:33.666 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.253:37139(127.21.111.253:37139)] connecting to peer
01:38:33.666 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.252:39629(127.21.111.252:39629)] connecting to peer
01:38:33.667 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.21.111.252:39629(127.21.111.252:39629)] Successfully connected to peer
01:38:33.667 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.21.111.253:37139(127.21.111.253:37139)] Successfully connected to peer
01:38:33.670 [DEBUG - kudu-nio-0] (SecurityContext.java:225) Refreshing Kerberos credentials...
01:38:33.670 [DEBUG - kudu-nio-0] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/krb5kdc/krb5cc
01:38:33.672 [DEBUG - kudu-nio-0] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
01:38:33.672 [WARN - kudu-nio-0] (SecurityContext.java:234) Tried to refresh Kerberos credentials but was unable to re-login from ticket cache
01:38:33.672 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.111.253:37139(127.21.111.253:37139)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:33.672 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.111.252:39629(127.21.111.252:39629)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:33.672 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.253:37139: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:33.672 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.252:39629: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:33.673 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629; will retry
01:38:33.673 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.21.111.253:37139(127.21.111.253:37139)] handling channelInactive
01:38:33.673 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.21.111.252:39629(127.21.111.252:39629)] handling channelInactive
01:38:33.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:33.674705 22615 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:33.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:33.668502 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:36630 (local address 127.21.111.253:37139)
01:38:33.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:33.669628 (+ 1126us) server_negotiation.cc:206] Beginning negotiation
01:38:33.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:33.669657 (+ 29us) server_negotiation.cc:399] Waiting for connection header
01:38:33.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:33.669698 (+ 41us) server_negotiation.cc:407] Connection header received
01:38:33.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:33.669879 (+ 181us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:33.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:33.669904 (+ 25us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:33.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:33.670120 (+ 216us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:33.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:33.671241 (+ 1121us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:33.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:33.673574 (+ 2333us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:36630: BlockingRecv error: recv got EOF from 127.0.0.1:36630 (error 108)
01:38:33.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":710,"thread_start_us":457,"threads_started":1}
01:38:33.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:33.675153 22614 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:33.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:33.667763 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:51472 (local address 127.21.111.252:39629)
01:38:33.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:33.669241 (+ 1478us) server_negotiation.cc:206] Beginning negotiation
01:38:33.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:33.669261 (+ 20us) server_negotiation.cc:399] Waiting for connection header
01:38:33.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:33.669307 (+ 46us) server_negotiation.cc:407] Connection header received
01:38:33.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:33.669481 (+ 174us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:33.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:33.669508 (+ 27us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:33.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:33.669723 (+ 215us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:33.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:33.670099 (+ 376us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:33.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:33.673952 (+ 3853us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:51472: BlockingRecv error: recv got EOF from 127.0.0.1:51472 (error 108)
01:38:33.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":1044,"thread_start_us":784,"threads_started":1}
01:38:34.205 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.253:37139(127.21.111.253:37139)] connecting to peer
01:38:34.206 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.252:39629(127.21.111.252:39629)] connecting to peer
01:38:34.207 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.21.111.253:37139(127.21.111.253:37139)] Successfully connected to peer
01:38:34.207 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.21.111.252:39629(127.21.111.252:39629)] Successfully connected to peer
01:38:34.211 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.111.252:39629(127.21.111.252:39629)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:34.211 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.111.253:37139(127.21.111.253:37139)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:34.211 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.252:39629: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:34.211 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.253:37139: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:34.211 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629; will retry
01:38:34.211 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.21.111.252:39629(127.21.111.252:39629)] handling channelInactive
01:38:34.212 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.21.111.253:37139(127.21.111.253:37139)] handling channelInactive
01:38:34.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:34.213102 22618 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:34.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:34.207934 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:51478 (local address 127.21.111.252:39629)
01:38:34.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:34.210066 (+ 2132us) server_negotiation.cc:206] Beginning negotiation
01:38:34.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:34.210088 (+ 22us) server_negotiation.cc:399] Waiting for connection header
01:38:34.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:34.210136 (+ 48us) server_negotiation.cc:407] Connection header received
01:38:34.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:34.210267 (+ 131us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:34.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:34.210305 (+ 38us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:34.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:34.210526 (+ 221us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:34.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:34.210881 (+ 355us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:34.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:34.212158 (+ 1277us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:51478: BlockingRecv error: recv got EOF from 127.0.0.1:51478 (error 108)
01:38:34.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":1735,"thread_start_us":1462,"threads_started":1}
01:38:34.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:34.213272 22619 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:34.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:34.208141 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:36636 (local address 127.21.111.253:37139)
01:38:34.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:34.210543 (+ 2402us) server_negotiation.cc:206] Beginning negotiation
01:38:34.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:34.210574 (+ 31us) server_negotiation.cc:399] Waiting for connection header
01:38:34.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:34.210619 (+ 45us) server_negotiation.cc:407] Connection header received
01:38:34.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:34.210766 (+ 147us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:34.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:34.210787 (+ 21us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:34.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:34.211020 (+ 233us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:34.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:34.211207 (+ 187us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:34.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:34.212484 (+ 1277us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:36636: BlockingRecv error: recv got EOF from 127.0.0.1:36636 (error 108)
01:38:34.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":1979,"thread_start_us":2177,"threads_started":1}
01:38:36.086 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.253:37139(127.21.111.253:37139)] connecting to peer
01:38:36.087 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.252:39629(127.21.111.252:39629)] connecting to peer
01:38:36.088 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.21.111.252:39629(127.21.111.252:39629)] Successfully connected to peer
01:38:36.089 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.21.111.253:37139(127.21.111.253:37139)] Successfully connected to peer
01:38:36.095 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.111.253:37139(127.21.111.253:37139)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:36.095 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.253:37139: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:36.096 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.21.111.253:37139(127.21.111.253:37139)] handling channelInactive
01:38:36.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:36.097533 22622 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:36.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:36.089113 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:36650 (local address 127.21.111.253:37139)
01:38:36.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:36.094168 (+ 5055us) server_negotiation.cc:206] Beginning negotiation
01:38:36.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:36.094204 (+ 36us) server_negotiation.cc:399] Waiting for connection header
01:38:36.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:36.094259 (+ 55us) server_negotiation.cc:407] Connection header received
01:38:36.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:36.094432 (+ 173us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:36.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:36.094457 (+ 25us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:36.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:36.094701 (+ 244us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:36.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:36.094929 (+ 228us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:36.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:36.096476 (+ 1547us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:36650: BlockingRecv error: recv got EOF from 127.0.0.1:36650 (error 108)
01:38:36.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":4601,"thread_start_us":4336,"threads_started":1}
01:38:36.105 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.111.252:39629(127.21.111.252:39629)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:36.106 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.252:39629: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:36.106 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629; will retry
01:38:36.106 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.21.111.252:39629(127.21.111.252:39629)] handling channelInactive
01:38:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:36.107527 22623 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:36.093553 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:51494 (local address 127.21.111.252:39629)
01:38:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:36.104488 (+ 10935us) server_negotiation.cc:206] Beginning negotiation
01:38:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:36.104503 (+ 15us) server_negotiation.cc:399] Waiting for connection header
01:38:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:36.104598 (+ 95us) server_negotiation.cc:407] Connection header received
01:38:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:36.104746 (+ 148us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:36.104764 (+ 18us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:36.104928 (+ 164us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:36.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:36.105113 (+ 185us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:36.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:36.106803 (+ 1690us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:51494: BlockingRecv error: recv got EOF from 127.0.0.1:51494 (error 108)
01:38:36.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":10547,"thread_start_us":10261,"threads_started":1}
01:38:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 01:38:38 dist-test-slave-37dq krb5kdc[21962](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790473118, etypes {rep=17 tkt=17 ses=17}, kudu/127.21.111.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:38:38.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:38.319495 22303 init.cc:295] Successfully reacquired a new kerberos TGT
01:38:39.206 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.253:37139(127.21.111.253:37139)] connecting to peer
01:38:39.207 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.252:39629(127.21.111.252:39629)] connecting to peer
01:38:39.207 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.21.111.253:37139(127.21.111.253:37139)] Successfully connected to peer
01:38:39.208 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.21.111.252:39629(127.21.111.252:39629)] Successfully connected to peer
01:38:39.211 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.111.253:37139(127.21.111.253:37139)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:39.211 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.111.252:39629(127.21.111.252:39629)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:39.211 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.253:37139: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:39.212 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.252:39629: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:39.212 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.21.111.253:37139(127.21.111.253:37139)] handling channelInactive
01:38:39.212 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.21.111.252:39629(127.21.111.252:39629)] handling channelInactive
01:38:39.212 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629; will retry
01:38:39.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:39.213513 22630 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:39.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:39.208079 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:36654 (local address 127.21.111.253:37139)
01:38:39.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:39.210433 (+ 2354us) server_negotiation.cc:206] Beginning negotiation
01:38:39.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:39.210459 (+ 26us) server_negotiation.cc:399] Waiting for connection header
01:38:39.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:39.210506 (+ 47us) server_negotiation.cc:407] Connection header received
01:38:39.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:39.210668 (+ 162us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:39.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:39.210683 (+ 15us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:39.210859 (+ 176us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:39.211064 (+ 205us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:39.212506 (+ 1442us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:36654: BlockingRecv error: recv got EOF from 127.0.0.1:36654 (error 108)
01:38:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":1818,"thread_start_us":1431,"threads_started":1}
01:38:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:39.213613 22631 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:39.209092 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:51502 (local address 127.21.111.252:39629)
01:38:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:39.210733 (+ 1641us) server_negotiation.cc:206] Beginning negotiation
01:38:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:39.210754 (+ 21us) server_negotiation.cc:399] Waiting for connection header
01:38:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:39.210802 (+ 48us) server_negotiation.cc:407] Connection header received
01:38:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:39.210936 (+ 134us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:39.210962 (+ 26us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:39.211175 (+ 213us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:39.211519 (+ 344us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:39.212777 (+ 1258us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:51502: BlockingRecv error: recv got EOF from 127.0.0.1:51502 (error 108)
01:38:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":1220,"thread_start_us":942,"threads_started":1}
01:38:39.805 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.253:37139(127.21.111.253:37139)] connecting to peer
01:38:39.806 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.252:39629(127.21.111.252:39629)] connecting to peer
01:38:39.807 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.21.111.252:39629(127.21.111.252:39629)] Successfully connected to peer
01:38:39.807 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.21.111.253:37139(127.21.111.253:37139)] Successfully connected to peer
01:38:39.816 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.111.253:37139(127.21.111.253:37139)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:39.817 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.253:37139: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:39.817 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.111.252:39629(127.21.111.252:39629)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:39.817 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.252:39629: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:39.817 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.21.111.253:37139(127.21.111.253:37139)] handling channelInactive
01:38:39.817 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629; will retry
01:38:39.818 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.21.111.252:39629(127.21.111.252:39629)] handling channelInactive
01:38:39.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:39.819125 22634 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:39.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:39.808134 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:36666 (local address 127.21.111.253:37139)
01:38:39.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:39.815796 (+ 7662us) server_negotiation.cc:206] Beginning negotiation
01:38:39.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:39.815816 (+ 20us) server_negotiation.cc:399] Waiting for connection header
01:38:39.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:39.815858 (+ 42us) server_negotiation.cc:407] Connection header received
01:38:39.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:39.815961 (+ 103us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:39.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:39.815977 (+ 16us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:39.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:39.816136 (+ 159us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:39.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:39.816334 (+ 198us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:39.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:39.817940 (+ 1606us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:36666: BlockingRecv error: recv got EOF from 127.0.0.1:36666 (error 108)
01:38:39.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":7224,"thread_start_us":6858,"threads_started":1}
01:38:39.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:39.819718 22633 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:39.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:39.807607 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:51510 (local address 127.21.111.252:39629)
01:38:39.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:39.816143 (+ 8536us) server_negotiation.cc:206] Beginning negotiation
01:38:39.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:39.816164 (+ 21us) server_negotiation.cc:399] Waiting for connection header
01:38:39.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:39.816209 (+ 45us) server_negotiation.cc:407] Connection header received
01:38:39.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:39.816333 (+ 124us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:39.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:39.816357 (+ 24us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:39.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:39.816584 (+ 227us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:39.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:39.817976 (+ 1392us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:39.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:39.818633 (+ 657us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:51510: BlockingRecv error: recv got EOF from 127.0.0.1:51510 (error 108)
01:38:39.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":8131,"thread_start_us":7862,"threads_started":1}
01:38:40.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 01:38:40 dist-test-slave-37dq krb5kdc[21962](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790473120, etypes {rep=17 tkt=17 ses=17}, kudu/127.21.111.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:38:40.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:40.441481 22395 init.cc:295] Successfully reacquired a new kerberos TGT
01:38:42.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 01:38:42 dist-test-slave-37dq krb5kdc[21962](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790473122, etypes {rep=17 tkt=17 ses=17}, kudu/127.21.111.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:38:42.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:42.754428 22499 init.cc:295] Successfully reacquired a new kerberos TGT
01:38:43.065 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.253:37139(127.21.111.253:37139)] connecting to peer
01:38:43.066 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.252:39629(127.21.111.252:39629)] connecting to peer
01:38:43.067 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.21.111.253:37139(127.21.111.253:37139)] Successfully connected to peer
01:38:43.067 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.21.111.252:39629(127.21.111.252:39629)] Successfully connected to peer
01:38:43.071 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.111.253:37139(127.21.111.253:37139)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:43.071 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.111.252:39629(127.21.111.252:39629)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:43.071 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.253:37139: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:43.071 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.252:39629: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:43.071 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629; will retry
01:38:43.071 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.21.111.253:37139(127.21.111.253:37139)] handling channelInactive
01:38:43.072 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.21.111.252:39629(127.21.111.252:39629)] handling channelInactive
01:38:43.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:43.073845 22638 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:43.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:43.066894 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:54116 (local address 127.21.111.253:37139)
01:38:43.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:43.070180 (+ 3286us) server_negotiation.cc:206] Beginning negotiation
01:38:43.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:43.070202 (+ 22us) server_negotiation.cc:399] Waiting for connection header
01:38:43.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:43.070238 (+ 36us) server_negotiation.cc:407] Connection header received
01:38:43.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:43.070343 (+ 105us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:43.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:43.070359 (+ 16us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:43.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:43.070516 (+ 157us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:43.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:43.070706 (+ 190us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:43.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:43.072503 (+ 1797us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:54116: BlockingRecv error: recv got EOF from 127.0.0.1:54116 (error 108)
01:38:43.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":1191,"thread_start_us":2318,"threads_started":1}
01:38:43.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:43.074088 22639 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:43.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:43.068058 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:34040 (local address 127.21.111.252:39629)
01:38:43.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:43.070341 (+ 2283us) server_negotiation.cc:206] Beginning negotiation
01:38:43.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:43.070361 (+ 20us) server_negotiation.cc:399] Waiting for connection header
01:38:43.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:43.070409 (+ 48us) server_negotiation.cc:407] Connection header received
01:38:43.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:43.070555 (+ 146us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:43.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:43.070588 (+ 33us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:43.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:43.070833 (+ 245us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:43.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:43.071049 (+ 216us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:43.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:43.072780 (+ 1731us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:34040: BlockingRecv error: recv got EOF from 127.0.0.1:34040 (error 108)
01:38:43.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":1790,"thread_start_us":1460,"threads_started":1}
01:38:44.126 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.253:37139(127.21.111.253:37139)] connecting to peer
01:38:44.127 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.252:39629(127.21.111.252:39629)] connecting to peer
01:38:44.127 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.21.111.252:39629(127.21.111.252:39629)] Successfully connected to peer
01:38:44.127 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.21.111.253:37139(127.21.111.253:37139)] Successfully connected to peer
01:38:44.135 [DEBUG - kudu-nio-1] (SecurityContext.java:225) Refreshing Kerberos credentials...
01:38:44.136 [DEBUG - kudu-nio-1] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskoFx69z/test-tmp/mini-kudu-cluster8016253420863221079/krb5kdc/krb5cc
01:38:44.137 [DEBUG - kudu-nio-1] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
01:38:44.137 [WARN - kudu-nio-1] (SecurityContext.java:234) Tried to refresh Kerberos credentials but was unable to re-login from ticket cache
01:38:44.137 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.111.252:39629(127.21.111.252:39629)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:44.137 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.111.253:37139(127.21.111.253:37139)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:44.137 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.253:37139: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:44.137 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.252:39629: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:44.138 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629; will retry
01:38:44.138 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.21.111.253:37139(127.21.111.253:37139)] handling channelInactive
01:38:44.138 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.21.111.252:39629(127.21.111.252:39629)] handling channelInactive
01:38:44.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:44.139783 22642 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:44.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:44.128694 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:54130 (local address 127.21.111.253:37139)
01:38:44.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:44.134601 (+ 5907us) server_negotiation.cc:206] Beginning negotiation
01:38:44.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:44.134672 (+ 71us) server_negotiation.cc:399] Waiting for connection header
01:38:44.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:44.134726 (+ 54us) server_negotiation.cc:407] Connection header received
01:38:44.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:44.134867 (+ 141us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:44.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:44.134894 (+ 27us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:44.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:44.135146 (+ 252us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:44.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:44.136136 (+ 990us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:44.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:44.138627 (+ 2491us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:54130: BlockingRecv error: recv got EOF from 127.0.0.1:54130 (error 108)
01:38:44.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":5473,"thread_start_us":5213,"threads_started":1}
01:38:44.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:44.140292 22643 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:44.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:44.128874 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:34042 (local address 127.21.111.252:39629)
01:38:44.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:44.135069 (+ 6195us) server_negotiation.cc:206] Beginning negotiation
01:38:44.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:44.135090 (+ 21us) server_negotiation.cc:399] Waiting for connection header
01:38:44.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:44.135134 (+ 44us) server_negotiation.cc:407] Connection header received
01:38:44.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:44.135271 (+ 137us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:44.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:44.135300 (+ 29us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:44.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:44.135541 (+ 241us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:44.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:44.136097 (+ 556us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:44.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:44.139099 (+ 3002us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:34042: BlockingRecv error: recv got EOF from 127.0.0.1:34042 (error 108)
01:38:44.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":5774,"thread_start_us":5480,"threads_started":1}
01:38:47.006 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.253:37139(127.21.111.253:37139)] connecting to peer
01:38:47.006 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.252:39629(127.21.111.252:39629)] connecting to peer
01:38:47.007 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.21.111.253:37139(127.21.111.253:37139)] Successfully connected to peer
01:38:47.007 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.21.111.252:39629(127.21.111.252:39629)] Successfully connected to peer
01:38:47.011 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.111.252:39629(127.21.111.252:39629)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:47.011 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.252:39629: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:47.012 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.21.111.252:39629(127.21.111.252:39629)] handling channelInactive
01:38:47.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:47.013970 22652 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:47.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:47.008568 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:34044 (local address 127.21.111.252:39629)
01:38:47.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:47.010576 (+ 2008us) server_negotiation.cc:206] Beginning negotiation
01:38:47.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:47.010591 (+ 15us) server_negotiation.cc:399] Waiting for connection header
01:38:47.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:47.010627 (+ 36us) server_negotiation.cc:407] Connection header received
01:38:47.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:47.010731 (+ 104us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:47.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:47.010746 (+ 15us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:47.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:47.010903 (+ 157us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:47.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:47.011095 (+ 192us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:47.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:47.012879 (+ 1784us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:34044: BlockingRecv error: recv got EOF from 127.0.0.1:34044 (error 108)
01:38:47.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":1706,"thread_start_us":1414,"threads_started":1}
01:38:47.016 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.111.253:37139(127.21.111.253:37139)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:47.016 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.253:37139: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:47.017 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629; will retry
01:38:47.017 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.21.111.253:37139(127.21.111.253:37139)] handling channelInactive
01:38:47.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:47.018484 22653 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:47.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:47.009350 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:54142 (local address 127.21.111.253:37139)
01:38:47.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:47.015605 (+ 6255us) server_negotiation.cc:206] Beginning negotiation
01:38:47.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:47.015628 (+ 23us) server_negotiation.cc:399] Waiting for connection header
01:38:47.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:47.015677 (+ 49us) server_negotiation.cc:407] Connection header received
01:38:47.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:47.015887 (+ 210us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:47.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:47.015902 (+ 15us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:47.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:47.016108 (+ 206us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:47.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:47.016281 (+ 173us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:47.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:47.017541 (+ 1260us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:54142: BlockingRecv error: recv got EOF from 127.0.0.1:54142 (error 108)
01:38:47.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":5865,"thread_start_us":5511,"threads_started":1}
01:38:48.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 01:38:48 dist-test-slave-37dq krb5kdc[21962](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790473128, etypes {rep=17 tkt=17 ses=17}, kudu/127.21.111.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:38:48.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:48.322069 22303 init.cc:295] Successfully reacquired a new kerberos TGT
01:38:50.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:50.242954 21377 debug-util.cc:398] Leaking SignalData structure 0x7b08000a0040 after lost signal to thread 21312
01:38:50.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:50.243633 21377 debug-util.cc:398] Leaking SignalData structure 0x7b08000a2320 after lost signal to thread 21380
01:38:50.407 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.253:37139(127.21.111.253:37139)] connecting to peer
01:38:50.408 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.21.111.252:39629(127.21.111.252:39629)] connecting to peer
01:38:50.409 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.21.111.253:37139(127.21.111.253:37139)] Successfully connected to peer
01:38:50.409 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.21.111.252:39629(127.21.111.252:39629)] Successfully connected to peer
01:38:50.423 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.111.252:39629(127.21.111.252:39629)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:50.423 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.252:39629: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:50.423 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.21.111.252:39629(127.21.111.252:39629)] handling channelInactive
01:38:50.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:50.425257 22658 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:50.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:50.410604 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:34056 (local address 127.21.111.252:39629)
01:38:50.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:50.422031 (+ 11427us) server_negotiation.cc:206] Beginning negotiation
01:38:50.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:50.422053 (+ 22us) server_negotiation.cc:399] Waiting for connection header
01:38:50.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:50.422093 (+ 40us) server_negotiation.cc:407] Connection header received
01:38:50.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:50.422199 (+ 106us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:50.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:50.422214 (+ 15us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:50.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:50.422382 (+ 168us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:50.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:50.423003 (+ 621us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:50.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:50.424147 (+ 1144us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:34056: BlockingRecv error: recv got EOF from 127.0.0.1:34056 (error 108)
01:38:50.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":10994,"thread_start_us":12089,"threads_started":1}
01:38:50.430 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.111.253:37139(127.21.111.253:37139)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:50.431 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.111.253:37139: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
01:38:50.431 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629; will retry
01:38:50.432 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=27508), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [4ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [100ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [118ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [126ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [127ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT),org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [149ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [152ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [153ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [153ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [168ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [171ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [172ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [172ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [190ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [193ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [196ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [197ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [224ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [231ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [232ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [232ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [260ms] refreshing cache from master, [261ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [262ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [262ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [269ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [273ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [276ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [276ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [300ms] refreshing cache from master, [301ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [301ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [301ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [305ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [306ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [311ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [311ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [420ms] refreshing cache from master, [420ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [421ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [421ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [428ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [428ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [436ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [436ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [540ms] refreshing cache from master, [540ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [541ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [541ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [551ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [554ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [554ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [554ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1040ms] refreshing cache from master, [1040ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [1041ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [1041ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [1048ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [1049ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1050ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1051ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1240ms] refreshing cache from master, [1241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [1241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [1241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [1246ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1246ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1247ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [1247ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [2680ms] refreshing cache from master, [2681ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [2681ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [2681ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [2685ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [2685ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [2687ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [2687ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [3300ms] refreshing cache from master, [3300ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [3301ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [3301ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [3304ms] too many traces: truncated at 100 traces, deferred=Deferred@1452771843(state=PENDING, result=null, callback=(continuation of Deferred@1439435013 after retry RPC after error@1694415169) -> (continuation of Deferred@2102598760 after retry RPC after error@394151507) -> (continuation of Deferred@584053144 after retry RPC after error@870998065) -> (continuation of Deferred@1455397496 after retry RPC after error@865468962) -> (continuation of Deferred@1846048505 after retry RPC after error@235104481) -> (continuation of Deferred@616914691 after retry RPC after error@667016875) -> (continuation of Deferred@181656340 after retry RPC after error@528674511) -> (continuation of Deferred@288458619 after retry RPC after error@942421872) -> (continuation of Deferred@741556250 after retry RPC after error@609167301) -> (continuation of Deferred@970483304 after retry RPC after error@1612071316) -> (continuation of Deferred@1452535324 after retry RPC after error@188713476) -> (continuation of Deferred@1898858029 after retry RPC after error@602317693) -> (continuation of Deferred@1783384595 after retry RPC after error@500466420) -> (continuation of Deferred@508076109 after retry RPC after error@144905650) -> (continuation of Deferred@12048999 after retry RPC after error@961362855) -> (continuation of Deferred@1650245131 after retry RPC after error@1916942711) -> (continuation of Deferred@1285075386 after retry RPC after error@475544461) -> (continuation of Deferred@460546795 after retry RPC after error@2023319923) -> (continuation of Deferred@1904546491 after retry RPC after error@744278869) -> (continuation of Deferred@750293007 after retry RPC after error@534357530) -> (continuation of Deferred@1183874960 after retry RPC after error@207360628) -> (continuation of Deferred@1678562072 after retry RPC after error@909594869) -> (continuation of Deferred@1939785052 after retry RPC after error@524134121) -> (continuation of Deferred@489370117 after retry RPC after error@286751149) -> (continuation of Deferred@346487542 after retry RPC after error@1725828197), errback=(continuation of Deferred@1439435013 after retry RPC after error@1694415169) -> (continuation of Deferred@2102598760 after retry RPC after error@394151507) -> (continuation of Deferred@584053144 after retry RPC after error@870998065) -> (continuation of Deferred@1455397496 after retry RPC after error@865468962) -> (continuation of Deferred@1846048505 after retry RPC after error@235104481) -> (continuation of Deferred@616914691 after retry RPC after error@667016875) -> (continuation of Deferred@181656340 after retry RPC after error@528674511) -> (continuation of Deferred@288458619 after retry RPC after error@942421872) -> (continuation of Deferred@741556250 after retry RPC after error@609167301) -> (continuation of Deferred@970483304 after retry RPC after error@1612071316) -> (continuation of Deferred@1452535324 after retry RPC after error@188713476) -> (continuation of Deferred@1898858029 after retry RPC after error@602317693) -> (continuation of Deferred@1783384595 after retry RPC after error@500466420) -> (continuation of Deferred@508076109 after retry RPC after error@144905650) -> (continuation of Deferred@12048999 after retry RPC after error@961362855) -> (continuation of Deferred@1650245131 after retry RPC after error@1916942711) -> (continuation of Deferred@1285075386 after retry RPC after error@475544461) -> (continuation of Deferred@460546795 after retry RPC after error@2023319923) -> (continuation of Deferred@1904546491 after retry RPC after error@744278869) -> (continuation of Deferred@750293007 after retry RPC after error@534357530) -> (continuation of Deferred@1183874960 after retry RPC after error@207360628) -> (continuation of Deferred@1678562072 after retry RPC after error@909594869) -> (continuation of Deferred@1939785052 after retry RPC after error@524134121) -> (continuation of Deferred@489370117 after retry RPC after error@286751149) -> (continuation of Deferred@346487542 after retry RPC after error@1725828197)))
01:38:50.435 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.21.111.253:37139(127.21.111.253:37139)] handling channelInactive
01:38:50.435 [DEBUG - main] (Connection.java:575) [peer master-127.21.111.253:37139(127.21.111.253:37139)] disconnecting while in state TERMINATED
01:38:50.436 [DEBUG - main] (Connection.java:575) [peer master-127.21.111.254:42895(127.21.111.254:42895)] disconnecting while in state READY
01:38:50.436 [DEBUG - main] (Connection.java:575) [peer master-127.21.111.252:39629(127.21.111.252:39629)] disconnecting while in state TERMINATED
01:38:50.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:50.437028 22657 negotiation.cc:337] Failed RPC negotiation. Trace:
01:38:50.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:50.409598 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:54144 (local address 127.21.111.253:37139)
01:38:50.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:50.429842 (+ 20244us) server_negotiation.cc:206] Beginning negotiation
01:38:50.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:50.429871 (+ 29us) server_negotiation.cc:399] Waiting for connection header
01:38:50.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:50.429909 (+ 38us) server_negotiation.cc:407] Connection header received
01:38:50.437 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
01:38:50.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:50.430017 (+ 108us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:38:50.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:50.430032 (+ 15us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:38:50.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:50.430275 (+ 243us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:38:50.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:50.430461 (+ 186us) server_negotiation.cc:226] Negotiated authn=SASL
01:38:50.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 01:38:50.435858 (+ 5397us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:54144: BlockingRecv error: recv got EOF from 127.0.0.1:54144 (error 108)
01:38:50.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":19951,"thread_start_us":21083,"threads_started":1}
01:38:50.438 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.21.111.254:42895(127.21.111.254:42895)] handling channelInactive
01:38:50.438 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.111.254:42895(127.21.111.254:42895)] cleaning up while in state READY due to: connection closed
01:38:50.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:50.440645 21951 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu with pid 22361
01:38:50.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:50.480567 21951 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu with pid 22289
01:38:50.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:50.513283 21951 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu with pid 22453
01:38:51.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:51.400997 21655 debug-util.cc:398] Leaking SignalData structure 0x7b08000b4400 after lost signal to thread 21526
01:38:51.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 01:38:51.402105 21655 debug-util.cc:398] Leaking SignalData structure 0x7b08000b9c00 after lost signal to thread 21658
01:38:51.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T01:38:51Z chronyd exiting
01:38:52.884 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
01:38:52.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:52.885972 21226 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu with pid 21507
01:38:52.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:52.914985 21226 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu with pid 21663
01:38:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:52.940150 21226 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu with pid 21807
01:38:52.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:52.966861 21226 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu with pid 21238
01:38:52.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:52.997511 21226 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu with pid 21308
01:38:53.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 01:38:53.028146 21226 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoFx69z/build/tsan/bin/kudu with pid 21383
01:38:54.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T01:38:54Z chronyd exiting
01:38:55.237 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testDoNotSwitchPrincipalsInExistingClient: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=27508), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [4ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [100ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [118ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [126ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [127ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT),org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [149ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [152ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [153ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [153ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [168ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [171ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [172ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [172ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [190ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [193ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [196ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [197ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [224ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [231ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [232ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [232ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [260ms] refreshing cache from master, [261ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [262ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [262ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [269ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [273ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [276ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [276ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [300ms] refreshing cache from master, [301ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [301ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [301ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [305ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [306ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [311ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [311ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [420ms] refreshing cache from master, [420ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [421ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [421ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [428ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [428ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [436ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [436ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [540ms] refreshing cache from master, [540ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [541ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [541ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [551ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [554ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [554ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [554ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1040ms] refreshing cache from master, [1040ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [1041ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [1041ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [1048ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [1049ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1050ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1051ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1240ms] refreshing cache from master, [1241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [1241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [1241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [1246ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1246ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1247ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [1247ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [2680ms] refreshing cache from master, [2681ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [2681ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [2681ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [2685ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [2685ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [2687ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [2687ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [3300ms] refreshing cache from master, [3300ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [3301ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [3301ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [3304ms] too many traces: truncated at 100 traces, deferred=Deferred@1452771843(state=PENDING, result=null, callback=(continuation of Deferred@1439435013 after retry RPC after error@1694415169) -> (continuation of Deferred@2102598760 after retry RPC after error@394151507) -> (continuation of Deferred@584053144 after retry RPC after error@870998065) -> (continuation of Deferred@1455397496 after retry RPC after error@865468962) -> (continuation of Deferred@1846048505 after retry RPC after error@235104481) -> (continuation of Deferred@616914691 after retry RPC after error@667016875) -> (continuation of Deferred@181656340 after retry RPC after error@528674511) -> (continuation of Deferred@288458619 after retry RPC after error@942421872) -> (continuation of Deferred@741556250 after retry RPC after error@609167301) -> (continuation of Deferred@970483304 after retry RPC after error@1612071316) -> (continuation of Deferred@1452535324 after retry RPC after error@188713476) -> (continuation of Deferred@1898858029 after retry RPC after error@602317693) -> (continuation of Deferred@1783384595 after retry RPC after error@500466420) -> (continuation of Deferred@508076109 after retry RPC after error@144905650) -> (continuation of Deferred@12048999 after retry RPC after error@961362855) -> (continuation of Deferred@1650245131 after retry RPC after error@1916942711) -> (continuation of Deferred@1285075386 after retry RPC after error@475544461) -> (continuation of Deferred@460546795 after retry RPC after error@2023319923) -> (continuation of Deferred@1904546491 after retry RPC after error@744278869) -> (continuation of Deferred@750293007 after retry RPC after error@534357530) -> (continuation of Deferred@1183874960 after retry RPC after error@207360628) -> (continuation of Deferred@1678562072 after retry RPC after error@909594869) -> (continuation of Deferred@1939785052 after retry RPC after error@524134121) -> (continuation of Deferred@489370117 after retry RPC after error@286751149) -> (continuation of Deferred@346487542 after retry RPC after error@1725828197), errback=(continuation of Deferred@1439435013 after retry RPC after error@1694415169) -> (continuation of Deferred@2102598760 after retry RPC after error@394151507) -> (continuation of Deferred@584053144 after retry RPC after error@870998065) -> (continuation of Deferred@1455397496 after retry RPC after error@865468962) -> (continuation of Deferred@1846048505 after retry RPC after error@235104481) -> (continuation of Deferred@616914691 after retry RPC after error@667016875) -> (continuation of Deferred@181656340 after retry RPC after error@528674511) -> (continuation of Deferred@288458619 after retry RPC after error@942421872) -> (continuation of Deferred@741556250 after retry RPC after error@609167301) -> (continuation of Deferred@970483304 after retry RPC after error@1612071316) -> (continuation of Deferred@1452535324 after retry RPC after error@188713476) -> (continuation of Deferred@1898858029 after retry RPC after error@602317693) -> (continuation of Deferred@1783384595 after retry RPC after error@500466420) -> (continuation of Deferred@508076109 after retry RPC after error@144905650) -> (continuation of Deferred@12048999 after retry RPC after error@961362855) -> (continuation of Deferred@1650245131 after retry RPC after error@1916942711) -> (continuation of Deferred@1285075386 after retry RPC after error@475544461) -> (continuation of Deferred@460546795 after retry RPC after error@2023319923) -> (continuation of Deferred@1904546491 after retry RPC after error@744278869) -> (continuation of Deferred@750293007 after retry RPC after error@534357530) -> (continuation of Deferred@1183874960 after retry RPC after error@207360628) -> (continuation of Deferred@1678562072 after retry RPC after error@909594869) -> (continuation of Deferred@1939785052 after retry RPC after error@524134121) -> (continuation of Deferred@489370117 after retry RPC after error@286751149) -> (continuation of Deferred@346487542 after retry RPC after error@1725828197)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386) ~[test/:?]
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testDoNotSwitchPrincipalsInExistingClient(TestSecurity.java:366) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565) ~[main/:?]
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=27508), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [4ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [100ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [118ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [126ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [127ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT),org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [149ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [152ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [153ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [153ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [168ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [171ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [172ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [172ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [190ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [193ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [196ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [197ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [224ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [231ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [232ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [232ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [260ms] refreshing cache from master, [261ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [262ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [262ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [269ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [273ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [276ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [276ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [300ms] refreshing cache from master, [301ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [301ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [301ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [305ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [306ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [311ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [311ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [420ms] refreshing cache from master, [420ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [421ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [421ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [428ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [428ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [436ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [436ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [540ms] refreshing cache from master, [540ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [541ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [541ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [551ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [554ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [554ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [554ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1040ms] refreshing cache from master, [1040ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [1041ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [1041ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [1048ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [1049ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1050ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1051ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1240ms] refreshing cache from master, [1241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [1241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [1241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [1246ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1246ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1247ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [1247ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [2680ms] refreshing cache from master, [2681ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [2681ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [2681ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [2685ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [2685ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [2687ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [2687ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [3300ms] refreshing cache from master, [3300ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [3301ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [3301ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [3304ms] too many traces: truncated at 100 traces, deferred=Deferred@1452771843(state=PENDING, result=null, callback=(continuation of Deferred@1439435013 after retry RPC after error@1694415169) -> (continuation of Deferred@2102598760 after retry RPC after error@394151507) -> (continuation of Deferred@584053144 after retry RPC after error@870998065) -> (continuation of Deferred@1455397496 after retry RPC after error@865468962) -> (continuation of Deferred@1846048505 after retry RPC after error@235104481) -> (continuation of Deferred@616914691 after retry RPC after error@667016875) -> (continuation of Deferred@181656340 after retry RPC after error@528674511) -> (continuation of Deferred@288458619 after retry RPC after error@942421872) -> (continuation of Deferred@741556250 after retry RPC after error@609167301) -> (continuation of Deferred@970483304 after retry RPC after error@1612071316) -> (continuation of Deferred@1452535324 after retry RPC after error@188713476) -> (continuation of Deferred@1898858029 after retry RPC after error@602317693) -> (continuation of Deferred@1783384595 after retry RPC after error@500466420) -> (continuation of Deferred@508076109 after retry RPC after error@144905650) -> (continuation of Deferred@12048999 after retry RPC after error@961362855) -> (continuation of Deferred@1650245131 after retry RPC after error@1916942711) -> (continuation of Deferred@1285075386 after retry RPC after error@475544461) -> (continuation of Deferred@460546795 after retry RPC after error@2023319923) -> (continuation of Deferred@1904546491 after retry RPC after error@744278869) -> (continuation of Deferred@750293007 after retry RPC after error@534357530) -> (continuation of Deferred@1183874960 after retry RPC after error@207360628) -> (continuation of Deferred@1678562072 after retry RPC after error@909594869) -> (continuation of Deferred@1939785052 after retry RPC after error@524134121) -> (continuation of Deferred@489370117 after retry RPC after error@286751149) -> (continuation of Deferred@346487542 after retry RPC after error@1725828197), errback=(continuation of Deferred@1439435013 after retry RPC after error@1694415169) -> (continuation of Deferred@2102598760 after retry RPC after error@394151507) -> (continuation of Deferred@584053144 after retry RPC after error@870998065) -> (continuation of Deferred@1455397496 after retry RPC after error@865468962) -> (continuation of Deferred@1846048505 after retry RPC after error@235104481) -> (continuation of Deferred@616914691 after retry RPC after error@667016875) -> (continuation of Deferred@181656340 after retry RPC after error@528674511) -> (continuation of Deferred@288458619 after retry RPC after error@942421872) -> (continuation of Deferred@741556250 after retry RPC after error@609167301) -> (continuation of Deferred@970483304 after retry RPC after error@1612071316) -> (continuation of Deferred@1452535324 after retry RPC after error@188713476) -> (continuation of Deferred@1898858029 after retry RPC after error@602317693) -> (continuation of Deferred@1783384595 after retry RPC after error@500466420) -> (continuation of Deferred@508076109 after retry RPC after error@144905650) -> (continuation of Deferred@12048999 after retry RPC after error@961362855) -> (continuation of Deferred@1650245131 after retry RPC after error@1916942711) -> (continuation of Deferred@1285075386 after retry RPC after error@475544461) -> (continuation of Deferred@460546795 after retry RPC after error@2023319923) -> (continuation of Deferred@1904546491 after retry RPC after error@744278869) -> (continuation of Deferred@750293007 after retry RPC after error@534357530) -> (continuation of Deferred@1183874960 after retry RPC after error@207360628) -> (continuation of Deferred@1678562072 after retry RPC after error@909594869) -> (continuation of Deferred@1939785052 after retry RPC after error@524134121) -> (continuation of Deferred@489370117 after retry RPC after error@286751149) -> (continuation of Deferred@346487542 after retry RPC after error@1725828197)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386)
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40)
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378)
at org.apache.kudu.client.TestSecurity.testDoNotSwitchPrincipalsInExistingClient(TestSecurity.java:366)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
... 30 more
01:38:55.588 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testDoNotSwitchPrincipalsInExistingClient: giving up after 1 attempts
01:38:55.588 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testDoNotSwitchPrincipalsInExistingClient: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=27508), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [4ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [100ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [118ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [126ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [127ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT),org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [149ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [152ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [153ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [153ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [168ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [171ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [172ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [172ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [190ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [193ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [196ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [197ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [224ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [231ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [232ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [232ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [260ms] refreshing cache from master, [261ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [262ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [262ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [269ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [273ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [276ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [276ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [300ms] refreshing cache from master, [301ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [301ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [301ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [305ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [306ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [311ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [311ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [420ms] refreshing cache from master, [420ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [421ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [421ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [428ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [428ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [436ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [436ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [540ms] refreshing cache from master, [540ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [541ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [541ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [551ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [554ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [554ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [554ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1040ms] refreshing cache from master, [1040ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [1041ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [1041ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [1048ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [1049ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1050ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1051ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1240ms] refreshing cache from master, [1241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [1241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [1241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [1246ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1246ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1247ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [1247ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [2680ms] refreshing cache from master, [2681ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [2681ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [2681ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [2685ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [2685ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [2687ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [2687ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [3300ms] refreshing cache from master, [3300ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [3301ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [3301ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [3304ms] too many traces: truncated at 100 traces, deferred=Deferred@1452771843(state=PENDING, result=null, callback=(continuation of Deferred@1439435013 after retry RPC after error@1694415169) -> (continuation of Deferred@2102598760 after retry RPC after error@394151507) -> (continuation of Deferred@584053144 after retry RPC after error@870998065) -> (continuation of Deferred@1455397496 after retry RPC after error@865468962) -> (continuation of Deferred@1846048505 after retry RPC after error@235104481) -> (continuation of Deferred@616914691 after retry RPC after error@667016875) -> (continuation of Deferred@181656340 after retry RPC after error@528674511) -> (continuation of Deferred@288458619 after retry RPC after error@942421872) -> (continuation of Deferred@741556250 after retry RPC after error@609167301) -> (continuation of Deferred@970483304 after retry RPC after error@1612071316) -> (continuation of Deferred@1452535324 after retry RPC after error@188713476) -> (continuation of Deferred@1898858029 after retry RPC after error@602317693) -> (continuation of Deferred@1783384595 after retry RPC after error@500466420) -> (continuation of Deferred@508076109 after retry RPC after error@144905650) -> (continuation of Deferred@12048999 after retry RPC after error@961362855) -> (continuation of Deferred@1650245131 after retry RPC after error@1916942711) -> (continuation of Deferred@1285075386 after retry RPC after error@475544461) -> (continuation of Deferred@460546795 after retry RPC after error@2023319923) -> (continuation of Deferred@1904546491 after retry RPC after error@744278869) -> (continuation of Deferred@750293007 after retry RPC after error@534357530) -> (continuation of Deferred@1183874960 after retry RPC after error@207360628) -> (continuation of Deferred@1678562072 after retry RPC after error@909594869) -> (continuation of Deferred@1939785052 after retry RPC after error@524134121) -> (continuation of Deferred@489370117 after retry RPC after error@286751149) -> (continuation of Deferred@346487542 after retry RPC after error@1725828197), errback=(continuation of Deferred@1439435013 after retry RPC after error@1694415169) -> (continuation of Deferred@2102598760 after retry RPC after error@394151507) -> (continuation of Deferred@584053144 after retry RPC after error@870998065) -> (continuation of Deferred@1455397496 after retry RPC after error@865468962) -> (continuation of Deferred@1846048505 after retry RPC after error@235104481) -> (continuation of Deferred@616914691 after retry RPC after error@667016875) -> (continuation of Deferred@181656340 after retry RPC after error@528674511) -> (continuation of Deferred@288458619 after retry RPC after error@942421872) -> (continuation of Deferred@741556250 after retry RPC after error@609167301) -> (continuation of Deferred@970483304 after retry RPC after error@1612071316) -> (continuation of Deferred@1452535324 after retry RPC after error@188713476) -> (continuation of Deferred@1898858029 after retry RPC after error@602317693) -> (continuation of Deferred@1783384595 after retry RPC after error@500466420) -> (continuation of Deferred@508076109 after retry RPC after error@144905650) -> (continuation of Deferred@12048999 after retry RPC after error@961362855) -> (continuation of Deferred@1650245131 after retry RPC after error@1916942711) -> (continuation of Deferred@1285075386 after retry RPC after error@475544461) -> (continuation of Deferred@460546795 after retry RPC after error@2023319923) -> (continuation of Deferred@1904546491 after retry RPC after error@744278869) -> (continuation of Deferred@750293007 after retry RPC after error@534357530) -> (continuation of Deferred@1183874960 after retry RPC after error@207360628) -> (continuation of Deferred@1678562072 after retry RPC after error@909594869) -> (continuation of Deferred@1939785052 after retry RPC after error@524134121) -> (continuation of Deferred@489370117 after retry RPC after error@286751149) -> (continuation of Deferred@346487542 after retry RPC after error@1725828197)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386) ~[test/:?]
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testDoNotSwitchPrincipalsInExistingClient(TestSecurity.java:366) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565) ~[main/:?]
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=27508), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [4ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [100ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [118ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [126ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [127ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT),org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [149ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [152ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [153ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [153ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [168ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [171ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [172ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [172ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [190ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [193ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [196ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [197ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [224ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [231ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [232ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [232ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [260ms] refreshing cache from master, [261ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [262ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [262ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [269ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [273ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [276ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [276ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [300ms] refreshing cache from master, [301ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [301ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [301ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [305ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [306ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [311ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [311ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [420ms] refreshing cache from master, [420ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [421ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [421ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [428ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [428ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [436ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [436ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [540ms] refreshing cache from master, [540ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [541ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [541ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [551ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [554ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [554ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [554ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1040ms] refreshing cache from master, [1040ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [1041ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [1041ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [1048ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [1049ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1050ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1051ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1240ms] refreshing cache from master, [1241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [1241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [1241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [1246ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1246ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1247ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [1247ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [2680ms] refreshing cache from master, [2681ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [2681ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [2681ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [2685ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.253:37139: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [2685ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.252:39629: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [2687ms] Sub RPC ConnectToMaster: received response from server master-127.21.111.254:42895: OK, [2687ms] delaying RPC due to: Service unavailable: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [3300ms] refreshing cache from master, [3300ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.253:37139, [3301ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.254:42895, [3301ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.111.252:39629, [3304ms] too many traces: truncated at 100 traces, deferred=Deferred@1452771843(state=PENDING, result=null, callback=(continuation of Deferred@1439435013 after retry RPC after error@1694415169) -> (continuation of Deferred@2102598760 after retry RPC after error@394151507) -> (continuation of Deferred@584053144 after retry RPC after error@870998065) -> (continuation of Deferred@1455397496 after retry RPC after error@865468962) -> (continuation of Deferred@1846048505 after retry RPC after error@235104481) -> (continuation of Deferred@616914691 after retry RPC after error@667016875) -> (continuation of Deferred@181656340 after retry RPC after error@528674511) -> (continuation of Deferred@288458619 after retry RPC after error@942421872) -> (continuation of Deferred@741556250 after retry RPC after error@609167301) -> (continuation of Deferred@970483304 after retry RPC after error@1612071316) -> (continuation of Deferred@1452535324 after retry RPC after error@188713476) -> (continuation of Deferred@1898858029 after retry RPC after error@602317693) -> (continuation of Deferred@1783384595 after retry RPC after error@500466420) -> (continuation of Deferred@508076109 after retry RPC after error@144905650) -> (continuation of Deferred@12048999 after retry RPC after error@961362855) -> (continuation of Deferred@1650245131 after retry RPC after error@1916942711) -> (continuation of Deferred@1285075386 after retry RPC after error@475544461) -> (continuation of Deferred@460546795 after retry RPC after error@2023319923) -> (continuation of Deferred@1904546491 after retry RPC after error@744278869) -> (continuation of Deferred@750293007 after retry RPC after error@534357530) -> (continuation of Deferred@1183874960 after retry RPC after error@207360628) -> (continuation of Deferred@1678562072 after retry RPC after error@909594869) -> (continuation of Deferred@1939785052 after retry RPC after error@524134121) -> (continuation of Deferred@489370117 after retry RPC after error@286751149) -> (continuation of Deferred@346487542 after retry RPC after error@1725828197), errback=(continuation of Deferred@1439435013 after retry RPC after error@1694415169) -> (continuation of Deferred@2102598760 after retry RPC after error@394151507) -> (continuation of Deferred@584053144 after retry RPC after error@870998065) -> (continuation of Deferred@1455397496 after retry RPC after error@865468962) -> (continuation of Deferred@1846048505 after retry RPC after error@235104481) -> (continuation of Deferred@616914691 after retry RPC after error@667016875) -> (continuation of Deferred@181656340 after retry RPC after error@528674511) -> (continuation of Deferred@288458619 after retry RPC after error@942421872) -> (continuation of Deferred@741556250 after retry RPC after error@609167301) -> (continuation of Deferred@970483304 after retry RPC after error@1612071316) -> (continuation of Deferred@1452535324 after retry RPC after error@188713476) -> (continuation of Deferred@1898858029 after retry RPC after error@602317693) -> (continuation of Deferred@1783384595 after retry RPC after error@500466420) -> (continuation of Deferred@508076109 after retry RPC after error@144905650) -> (continuation of Deferred@12048999 after retry RPC after error@961362855) -> (continuation of Deferred@1650245131 after retry RPC after error@1916942711) -> (continuation of Deferred@1285075386 after retry RPC after error@475544461) -> (continuation of Deferred@460546795 after retry RPC after error@2023319923) -> (continuation of Deferred@1904546491 after retry RPC after error@744278869) -> (continuation of Deferred@750293007 after retry RPC after error@534357530) -> (continuation of Deferred@1183874960 after retry RPC after error@207360628) -> (continuation of Deferred@1678562072 after retry RPC after error@909594869) -> (continuation of Deferred@1939785052 after retry RPC after error@524134121) -> (continuation of Deferred@489370117 after retry RPC after error@286751149) -> (continuation of Deferred@346487542 after retry RPC after error@1725828197)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386)
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40)
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378)
at org.apache.kudu.client.TestSecurity.testDoNotSwitchPrincipalsInExistingClient(TestSecurity.java:366)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.21.111.253:37139,127.21.111.254:42895,127.21.111.252:39629) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
... 30 more