22:18:13.220 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
22:18:13.220 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskyO6VHJ/test-tmp
22:18:13.220 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
22:18:13.220 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskyO6VHJ/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
22:18:13.222 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
22:18:13.524 [DEBUG - main] (MiniKuduCluster.java:195) Response:
22:18:13.525 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
22:18:13.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:13.527671 22820 mini_oidc.cc:152] Starting JWKS server
22:18:13.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:13.530889 22820 webserver.cc:460] Webserver started at http://127.0.0.1:46759/ using document root <none> and password file <none>
22:18:13.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:13.531260 22820 mini_oidc.cc:189] Starting OIDC Discovery server
22:18:13.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:13.532918 22820 webserver.cc:460] Webserver started at http://127.0.0.1:41607/ using document root <none> and password file <none>
22:18:13.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T22:18:13Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
22:18:13.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T22:18:13Z Disabled control of system clock
22:18:13.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:13.579588 22820 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:18:13.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:18:13.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-0/wal
22:18:13.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-0/data
22:18:13.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
22:18:13.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
22:18:13.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
22:18:13.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:18:13.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:18:13.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:18:13.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-0/logs
22:18:13.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-0/data/info.pb
22:18:13.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:18:13.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
22:18:13.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
22:18:13.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
22:18:13.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
22:18:13.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:18:13.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:18:13.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:18:13.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
22:18:13.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
22:18:13.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:18:13.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:18:13.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.73.62:42139
22:18:13.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.73.62
22:18:13.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:18:13.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.73.20:34075
22:18:13.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:18:13.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:18:13.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:18:13.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:18:13.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.73.62:42139,127.22.73.61:36531,127.22.73.60:38817
22:18:13.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:18:13.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46759/jwks.json with env {}
22:18:13.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:13.863924 22832 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
22:18:13.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:13.864425 22832 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
22:18:13.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:13.864877 22832 flags.cc:425] Enabled unsafe flag: --never_fsync=true
22:18:13.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:13.894663 22832 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
22:18:13.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:13.894915 22832 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
22:18:13.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:13.895112 22832 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
22:18:13.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:13.895298 22832 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
22:18:13.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:13.895489 22832 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
22:18:13.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:13.895620 22832 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46759/jwks.json
22:18:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:13.930152 22832 master_runner.cc:386] Master server non-default flags:
22:18:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:18:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.73.20:34075
22:18:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:18:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:18:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-0/data
22:18:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-0/wal
22:18:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:18:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.73.62:42139,127.22.73.61:36531,127.22.73.60:38817
22:18:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:18:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:18:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:18:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.73.62:42139
22:18:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:18:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
22:18:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:18:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46759/jwks.json
22:18:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:18:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:18:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-0/data/info.pb
22:18:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.73.62
22:18:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:18:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
22:18:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:18:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
22:18:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
22:18:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:18:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-0/logs
22:18:13.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:18:13.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
22:18:13.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:18:13.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
22:18:13.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
22:18:13.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
22:18:13.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
22:18:13.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 21:22:24 UTC on ccycloud.quasar-wysbne.root.comops.site
22:18:13.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676318
22:18:13.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
22:18:13.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:13.931308 22832 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
22:18:13.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:13.932864 22832 file_cache.cc:493] Constructed file cache file cache with capacity 419430
22:18:13.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:13.949879 22838 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:18:15.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:13.949955 22841 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:18:15.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:13.949963 22839 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:18:15.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:13.951081 22832 server_base.cc:1034] running on GCE node
22:18:15.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:15.113425 22832 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
22:18:15.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:15.115940 22832 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
22:18:15.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:15.117264 22832 hybrid_clock.cc:648] HybridClock initialized: now 1790547495117230 us; error 55 us; skew 500 ppm
22:18:15.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:15.118067 22832 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
22:18:15.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:15.131888 22832 webserver.cc:460] Webserver started at http://127.22.73.62:33791/ using document root <none> and password file <none>
22:18:15.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:15.132840 22832 fs_manager.cc:359] Metadata directory not provided
22:18:15.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:15.133019 22832 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:18:15.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:15.133409 22832 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
22:18:15.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:15.137966 22832 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-0/data/instance:
22:18:15.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f65d617dc0f64ab2b6a29b2dc4b84875"
22:18:15.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:18:15 on dist-test-slave-qzth"
22:18:15.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:15.138969 22832 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-0/wal/instance:
22:18:15.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f65d617dc0f64ab2b6a29b2dc4b84875"
22:18:15.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:18:15 on dist-test-slave-qzth"
22:18:15.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:15.145596 22832 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.006s sys 0.001s
22:18:15.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:15.150887 22848 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
22:18:15.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:15.151834 22832 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.002s
22:18:15.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:15.152117 22832 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-0/data,/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-0/wal
22:18:15.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f65d617dc0f64ab2b6a29b2dc4b84875"
22:18:15.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:18:15 on dist-test-slave-qzth"
22:18:15.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:15.152390 22832 fs_report.cc:383] FS layout report
22:18:15.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
22:18:15.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-0/wal
22:18:15.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-0/wal
22:18:15.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-0/data/data
22:18:15.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
22:18:15.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
22:18:15.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
22:18:15.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
22:18:15.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
22:18:15.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
22:18:15.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
22:18:15.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
22:18:15.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
22:18:15.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
22:18:15.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
22:18:15.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:15.196413 22832 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
22:18:15.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:15.197845 22832 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
22:18:15.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:15.198225 22832 kserver.cc:163] Server-wide thread pool size limit: 3276
22:18:15.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:15.264969 22832 rpc_server.cc:307] RPC server started. Bound to: 127.22.73.62:42139
22:18:15.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:15.265055 22899 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.73.62:42139 every 8 connection(s)
22:18:15.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:15.267551 22832 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-0/data/info.pb
22:18:15.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:15.272739 22900 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
22:18:15.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:15.273469 22820 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu as pid 22832
22:18:15.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:15.273957 22820 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-0/wal/instance
22:18:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:15.277199 22820 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:18:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:18:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-1/wal
22:18:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-1/data
22:18:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
22:18:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
22:18:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
22:18:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:18:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:18:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:18:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-1/logs
22:18:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-1/data/info.pb
22:18:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:18:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
22:18:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
22:18:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
22:18:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
22:18:15.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:18:15.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:18:15.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:18:15.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
22:18:15.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
22:18:15.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:18:15.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:18:15.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.73.61:36531
22:18:15.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.73.61
22:18:15.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:18:15.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.73.20:34075
22:18:15.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:18:15.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:18:15.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:18:15.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:18:15.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.73.62:42139,127.22.73.61:36531,127.22.73.60:38817
22:18:15.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:18:15.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46759/jwks.json with env {}
22:18:15.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:15.283483 22900 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.73.62" port: 42139 } has no permanent_uuid. Determining permanent_uuid...
22:18:15.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:15.313055 22900 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.73.61" port: 36531 } has no permanent_uuid. Determining permanent_uuid...
22:18:15.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:15.315243 22851 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.73.61:36531: connect: Connection refused (error 111)
22:18:15.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:15.319260 22900 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.73.61:36531: Network error: Client connection negotiation failed: client connection to 127.22.73.61:36531: connect: Connection refused (error 111)
22:18:15.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:15.368678 22900 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.73.61" port: 36531 } attempt: 1
22:18:15.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:15.372520 22900 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.73.61:36531: Network error: Client connection negotiation failed: client connection to 127.22.73.61:36531: connect: Connection refused (error 111)
22:18:15.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:15.440878 22900 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.73.61" port: 36531 } attempt: 2
22:18:15.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:15.444674 22900 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.73.61:36531: Network error: Client connection negotiation failed: client connection to 127.22.73.61:36531: connect: Connection refused (error 111)
22:18:15.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:15.536023 22900 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.73.61" port: 36531 } attempt: 3
22:18:15.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:15.539577 22900 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.73.61:36531: Network error: Client connection negotiation failed: client connection to 127.22.73.61:36531: connect: Connection refused (error 111)
22:18:15.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:15.576650 22902 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
22:18:15.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:15.577270 22902 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
22:18:15.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:15.577859 22902 flags.cc:425] Enabled unsafe flag: --never_fsync=true
22:18:15.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:15.607591 22902 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
22:18:15.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:15.607950 22902 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
22:18:15.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:15.608256 22902 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
22:18:15.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:15.608548 22902 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
22:18:15.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:15.608850 22902 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
22:18:15.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:15.609069 22902 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46759/jwks.json
22:18:15.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:15.642048 22902 master_runner.cc:386] Master server non-default flags:
22:18:15.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:18:15.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.73.20:34075
22:18:15.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:18:15.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:18:15.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-1/data
22:18:15.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-1/wal
22:18:15.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:18:15.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.73.62:42139,127.22.73.61:36531,127.22.73.60:38817
22:18:15.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:18:15.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:18:15.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:18:15.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.73.61:36531
22:18:15.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:18:15.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
22:18:15.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:18:15.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46759/jwks.json
22:18:15.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:18:15.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:18:15.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-1/data/info.pb
22:18:15.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.73.61
22:18:15.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:18:15.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
22:18:15.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:18:15.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
22:18:15.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
22:18:15.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:18:15.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-1/logs
22:18:15.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:18:15.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
22:18:15.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:18:15.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
22:18:15.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
22:18:15.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
22:18:15.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
22:18:15.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 21:22:24 UTC on ccycloud.quasar-wysbne.root.comops.site
22:18:15.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676318
22:18:15.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
22:18:15.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:15.643522 22902 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
22:18:15.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:15.645136 22902 file_cache.cc:493] Constructed file cache file cache with capacity 419430
22:18:15.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:15.661705 22912 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:18:15.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:15.682996 22900 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.73.61" port: 36531 } attempt: 4
22:18:15.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:15.687132 22900 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.73.61:36531: Network error: Client connection negotiation failed: client connection to 127.22.73.61:36531: connect: Connection refused (error 111)
22:18:15.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:15.986726 22900 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.73.61" port: 36531 } attempt: 5
22:18:15.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:15.991588 22900 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.73.61:36531: Network error: Client connection negotiation failed: client connection to 127.22.73.61:36531: connect: Connection refused (error 111)
22:18:16.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:16.539023 22900 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.73.61" port: 36531 } attempt: 6
22:18:16.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:16.545122 22900 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.73.61:36531: Network error: Client connection negotiation failed: client connection to 127.22.73.61:36531: connect: Connection refused (error 111)
22:18:16.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:15.661753 22911 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:18:16.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:15.661715 22914 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:18:16.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:15.663237 22902 server_base.cc:1034] running on GCE node
22:18:16.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:16.842135 22902 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
22:18:16.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:16.844631 22902 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
22:18:16.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:16.845937 22902 hybrid_clock.cc:648] HybridClock initialized: now 1790547496845920 us; error 55 us; skew 500 ppm
22:18:16.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:16.846721 22902 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
22:18:16.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:16.852536 22902 webserver.cc:460] Webserver started at http://127.22.73.61:45835/ using document root <none> and password file <none>
22:18:16.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:16.853344 22902 fs_manager.cc:359] Metadata directory not provided
22:18:16.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:16.853513 22902 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:18:16.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:16.853994 22902 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
22:18:16.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:16.858669 22902 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-1/data/instance:
22:18:16.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a5ed30b572d44c96a73bcc696ea0282f"
22:18:16.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:18:16 on dist-test-slave-qzth"
22:18:16.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:16.859661 22902 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-1/wal/instance:
22:18:16.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a5ed30b572d44c96a73bcc696ea0282f"
22:18:16.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:18:16 on dist-test-slave-qzth"
22:18:16.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:16.866348 22902 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.007s sys 0.001s
22:18:16.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:16.871539 22923 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
22:18:16.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:16.872403 22902 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.005s sys 0.000s
22:18:16.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:16.872695 22902 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-1/data,/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-1/wal
22:18:16.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a5ed30b572d44c96a73bcc696ea0282f"
22:18:16.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:18:16 on dist-test-slave-qzth"
22:18:16.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:16.872964 22902 fs_report.cc:383] FS layout report
22:18:16.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
22:18:16.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-1/wal
22:18:16.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-1/wal
22:18:16.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-1/data/data
22:18:16.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
22:18:16.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
22:18:16.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
22:18:16.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
22:18:16.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
22:18:16.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
22:18:16.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
22:18:16.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
22:18:16.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
22:18:16.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
22:18:16.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
22:18:16.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:16.908685 22902 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
22:18:16.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:16.910002 22902 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
22:18:16.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:16.910375 22902 kserver.cc:163] Server-wide thread pool size limit: 3276
22:18:16.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:16.979372 22902 rpc_server.cc:307] RPC server started. Bound to: 127.22.73.61:36531
22:18:16.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:16.979470 22974 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.73.61:36531 every 8 connection(s)
22:18:16.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:16.982134 22902 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-1/data/info.pb
22:18:16.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:16.986862 22975 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
22:18:16.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:16.992025 22820 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu as pid 22902
22:18:16.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:16.992359 22820 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-1/wal/instance
22:18:16.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:16.994220 22820 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:18:16.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:18:16.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-2/wal
22:18:16.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-2/data
22:18:16.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
22:18:16.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
22:18:16.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
22:18:16.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:18:16.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:18:16.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:18:16.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-2/logs
22:18:16.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-2/data/info.pb
22:18:16.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:18:16.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
22:18:16.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
22:18:16.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
22:18:16.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
22:18:16.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:18:16.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:18:16.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:18:16.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
22:18:16.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
22:18:16.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:18:16.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:18:16.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.73.60:38817
22:18:16.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.73.60
22:18:16.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:18:16.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.73.20:34075
22:18:16.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:18:16.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:18:16.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:18:16.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:18:16.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.73.62:42139,127.22.73.61:36531,127.22.73.60:38817
22:18:16.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:18:16.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46759/jwks.json with env {}
22:18:16.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:16.997812 22975 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.73.62" port: 42139 } has no permanent_uuid. Determining permanent_uuid...
22:18:17.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:17.018352 22975 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.73.61" port: 36531 } has no permanent_uuid. Determining permanent_uuid...
22:18:17.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:17.033249 22975 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.73.60" port: 38817 } has no permanent_uuid. Determining permanent_uuid...
22:18:17.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:17.035334 22924 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.73.60:38817: connect: Connection refused (error 111)
22:18:17.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:17.038826 22975 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.73.60:38817: Network error: Client connection negotiation failed: client connection to 127.22.73.60:38817: connect: Connection refused (error 111)
22:18:17.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:17.088323 22975 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.73.60" port: 38817 } attempt: 1
22:18:17.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:17.091769 22975 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.73.60:38817: Network error: Client connection negotiation failed: client connection to 127.22.73.60:38817: connect: Connection refused (error 111)
22:18:17.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:17.160120 22975 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.73.60" port: 38817 } attempt: 2
22:18:17.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:17.163432 22975 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.73.60:38817: Network error: Client connection negotiation failed: client connection to 127.22.73.60:38817: connect: Connection refused (error 111)
22:18:17.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:17.254761 22975 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.73.60" port: 38817 } attempt: 3
22:18:17.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:17.258213 22975 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.73.60:38817: Network error: Client connection negotiation failed: client connection to 127.22.73.60:38817: connect: Connection refused (error 111)
22:18:17.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:17.284667 22977 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
22:18:17.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:17.285144 22977 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
22:18:17.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:17.285576 22977 flags.cc:425] Enabled unsafe flag: --never_fsync=true
22:18:17.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:17.315523 22977 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
22:18:17.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:17.315768 22977 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
22:18:17.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:17.315955 22977 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
22:18:17.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:17.316129 22977 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
22:18:17.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:17.316303 22977 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
22:18:17.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:17.316453 22977 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46759/jwks.json
22:18:17.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:17.348706 22977 master_runner.cc:386] Master server non-default flags:
22:18:17.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:18:17.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.73.20:34075
22:18:17.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:18:17.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:18:17.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-2/data
22:18:17.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-2/wal
22:18:17.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:18:17.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.73.62:42139,127.22.73.61:36531,127.22.73.60:38817
22:18:17.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:18:17.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:18:17.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:18:17.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.73.60:38817
22:18:17.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:18:17.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
22:18:17.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:18:17.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46759/jwks.json
22:18:17.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:18:17.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:18:17.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-2/data/info.pb
22:18:17.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.73.60
22:18:17.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:18:17.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
22:18:17.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:18:17.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
22:18:17.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
22:18:17.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:18:17.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-2/logs
22:18:17.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:18:17.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
22:18:17.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:18:17.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
22:18:17.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
22:18:17.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
22:18:17.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
22:18:17.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 21:22:24 UTC on ccycloud.quasar-wysbne.root.comops.site
22:18:17.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676318
22:18:17.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
22:18:17.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:17.349876 22977 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
22:18:17.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:17.351320 22977 file_cache.cc:493] Constructed file cache file cache with capacity 419430
22:18:17.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:17.368724 22990 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:18:17.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:17.401659 22975 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.73.60" port: 38817 } attempt: 4
22:18:17.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:17.405319 22975 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.73.60:38817: Network error: Client connection negotiation failed: client connection to 127.22.73.60:38817: connect: Connection refused (error 111)
22:18:17.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:17.605713 22900 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.73.61" port: 36531 } attempt: 7
22:18:17.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:17.622370 22900 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.73.60" port: 38817 } has no permanent_uuid. Determining permanent_uuid...
22:18:17.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:17.628393 22900 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.73.60:38817: Network error: Client connection negotiation failed: client connection to 127.22.73.60:38817: connect: Connection refused (error 111)
22:18:17.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:17.686759 22900 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.73.60" port: 38817 } attempt: 1
22:18:17.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:17.690562 22900 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.73.60:38817: Network error: Client connection negotiation failed: client connection to 127.22.73.60:38817: connect: Connection refused (error 111)
22:18:17.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:17.704850 22975 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.73.60" port: 38817 } attempt: 5
22:18:17.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:17.708349 22975 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.73.60:38817: Network error: Client connection negotiation failed: client connection to 127.22.73.60:38817: connect: Connection refused (error 111)
22:18:17.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:17.772149 22900 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.73.60" port: 38817 } attempt: 2
22:18:17.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:17.776439 22900 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.73.60:38817: Network error: Client connection negotiation failed: client connection to 127.22.73.60:38817: connect: Connection refused (error 111)
22:18:17.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:17.861860 22900 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.73.60" port: 38817 } attempt: 3
22:18:17.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:17.865679 22900 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.73.60:38817: Network error: Client connection negotiation failed: client connection to 127.22.73.60:38817: connect: Connection refused (error 111)
22:18:18.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.006088 22900 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.73.60" port: 38817 } attempt: 4
22:18:18.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:18.009979 22900 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.73.60:38817: Network error: Client connection negotiation failed: client connection to 127.22.73.60:38817: connect: Connection refused (error 111)
22:18:18.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.255920 22975 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.73.60" port: 38817 } attempt: 6
22:18:18.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:18.264372 22975 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.73.60:38817: Network error: Client connection negotiation failed: client connection to 127.22.73.60:38817: connect: Connection refused (error 111)
22:18:18.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.293331 22900 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.73.60" port: 38817 } attempt: 5
22:18:18.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:18.296597 22900 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.73.60:38817: Network error: Client connection negotiation failed: client connection to 127.22.73.60:38817: connect: Connection refused (error 111)
22:18:18.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:17.368737 22987 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:18:18.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:17.368861 22988 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:18:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:18.526945 22989 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
22:18:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.527082 22977 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
22:18:18.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.531172 22977 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
22:18:18.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.534138 22977 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
22:18:18.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.535573 22977 hybrid_clock.cc:648] HybridClock initialized: now 1790547498535554 us; error 48 us; skew 500 ppm
22:18:18.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.536439 22977 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
22:18:18.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.542862 22977 webserver.cc:460] Webserver started at http://127.22.73.60:39279/ using document root <none> and password file <none>
22:18:18.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.543747 22977 fs_manager.cc:359] Metadata directory not provided
22:18:18.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.543917 22977 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:18:18.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.544288 22977 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
22:18:18.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.549065 22977 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-2/data/instance:
22:18:18.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2d5ca51e59ac4b5f97da5e6c0d50f839"
22:18:18.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:18:18 on dist-test-slave-qzth"
22:18:18.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.550161 22977 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-2/wal/instance:
22:18:18.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2d5ca51e59ac4b5f97da5e6c0d50f839"
22:18:18.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:18:18 on dist-test-slave-qzth"
22:18:18.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.556783 22977 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.006s sys 0.000s
22:18:18.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.562633 23002 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
22:18:18.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.563822 22977 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
22:18:18.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.564126 22977 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-2/data,/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-2/wal
22:18:18.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2d5ca51e59ac4b5f97da5e6c0d50f839"
22:18:18.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:18:18 on dist-test-slave-qzth"
22:18:18.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.564566 22977 fs_report.cc:383] FS layout report
22:18:18.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
22:18:18.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-2/wal
22:18:18.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-2/wal
22:18:18.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-2/data/data
22:18:18.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
22:18:18.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
22:18:18.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
22:18:18.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
22:18:18.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
22:18:18.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
22:18:18.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
22:18:18.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
22:18:18.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
22:18:18.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
22:18:18.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
22:18:18.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.596378 22977 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
22:18:18.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.597630 22977 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
22:18:18.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.598070 22977 kserver.cc:163] Server-wide thread pool size limit: 3276
22:18:18.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.664113 22977 rpc_server.cc:307] RPC server started. Bound to: 127.22.73.60:38817
22:18:18.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.664204 23053 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.73.60:38817 every 8 connection(s)
22:18:18.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.666800 22977 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-2/data/info.pb
22:18:18.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.668799 22820 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu as pid 22977
22:18:18.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.669312 22820 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/master-2/wal/instance
22:18:18.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.672302 23054 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
22:18:18.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.691860 23054 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.73.62" port: 42139 } has no permanent_uuid. Determining permanent_uuid...
22:18:18.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.715348 23054 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.73.61" port: 36531 } has no permanent_uuid. Determining permanent_uuid...
22:18:18.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.725400 23054 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.73.60" port: 38817 } has no permanent_uuid. Determining permanent_uuid...
22:18:18.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.741065 23054 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2d5ca51e59ac4b5f97da5e6c0d50f839: Bootstrap starting.
22:18:18.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.747234 23054 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 2d5ca51e59ac4b5f97da5e6c0d50f839: Neither blocks nor log segments found. Creating new log.
22:18:18.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.748819 23054 log.cc:824] T 00000000000000000000000000000000 P 2d5ca51e59ac4b5f97da5e6c0d50f839: Log is configured to *not* fsync() on all Append() calls
22:18:18.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.753186 23054 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2d5ca51e59ac4b5f97da5e6c0d50f839: No bootstrap required, opened a new log
22:18:18.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.773202 23054 raft_consensus.cc:348] T 00000000000000000000000000000000 P 2d5ca51e59ac4b5f97da5e6c0d50f839 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f65d617dc0f64ab2b6a29b2dc4b84875" member_type: VOTER last_known_addr { host: "127.22.73.62" port: 42139 } } peers { permanent_uuid: "a5ed30b572d44c96a73bcc696ea0282f" member_type: VOTER last_known_addr { host: "127.22.73.61" port: 36531 } } peers { permanent_uuid: "2d5ca51e59ac4b5f97da5e6c0d50f839" member_type: VOTER last_known_addr { host: "127.22.73.60" port: 38817 } }
22:18:18.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.773751 23054 raft_consensus.cc:374] T 00000000000000000000000000000000 P 2d5ca51e59ac4b5f97da5e6c0d50f839 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
22:18:18.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.774034 23054 raft_consensus.cc:729] T 00000000000000000000000000000000 P 2d5ca51e59ac4b5f97da5e6c0d50f839 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2d5ca51e59ac4b5f97da5e6c0d50f839, State: Initialized, Role: FOLLOWER
22:18:18.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.774724 23054 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2d5ca51e59ac4b5f97da5e6c0d50f839 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f65d617dc0f64ab2b6a29b2dc4b84875" member_type: VOTER last_known_addr { host: "127.22.73.62" port: 42139 } } peers { permanent_uuid: "a5ed30b572d44c96a73bcc696ea0282f" member_type: VOTER last_known_addr { host: "127.22.73.61" port: 36531 } } peers { permanent_uuid: "2d5ca51e59ac4b5f97da5e6c0d50f839" member_type: VOTER last_known_addr { host: "127.22.73.60" port: 38817 } }
22:18:18.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.776909 23063 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2d5ca51e59ac4b5f97da5e6c0d50f839 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f65d617dc0f64ab2b6a29b2dc4b84875" member_type: VOTER last_known_addr { host: "127.22.73.62" port: 42139 } } peers { permanent_uuid: "a5ed30b572d44c96a73bcc696ea0282f" member_type: VOTER last_known_addr { host: "127.22.73.61" port: 36531 } } peers { permanent_uuid: "2d5ca51e59ac4b5f97da5e6c0d50f839" member_type: VOTER last_known_addr { host: "127.22.73.60" port: 38817 } } }
22:18:18.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.777508 23063 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2d5ca51e59ac4b5f97da5e6c0d50f839 [sys.catalog]: This master's current role is: FOLLOWER
22:18:18.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.778450 23054 sys_catalog.cc:564] T 00000000000000000000000000000000 P 2d5ca51e59ac4b5f97da5e6c0d50f839 [sys.catalog]: configured and running, proceeding with master startup.
22:18:18.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:18.802874 23074 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2d5ca51e59ac4b5f97da5e6c0d50f839: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:18:18.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:18.803157 23074 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:18:18.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.799059 22977 master_runner.cc:186] Error getting master registration for 127.22.73.62:42139: OK, instance_id { permanent_uuid: "f65d617dc0f64ab2b6a29b2dc4b84875" instance_seqno: 1790547495246870 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
22:18:18.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.820261 22977 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
22:18:18.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.849128 22900 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.73.60" port: 38817 } attempt: 6
22:18:18.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:18.852674 23008 tablet.cc:2369] T 00000000000000000000000000000000 P 2d5ca51e59ac4b5f97da5e6c0d50f839: Can't schedule compaction. Clean time has not been advanced past its initial value.
22:18:18.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.867535 22900 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f65d617dc0f64ab2b6a29b2dc4b84875: Bootstrap starting.
22:18:18.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.872665 22900 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f65d617dc0f64ab2b6a29b2dc4b84875: Neither blocks nor log segments found. Creating new log.
22:18:18.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.874428 22900 log.cc:824] T 00000000000000000000000000000000 P f65d617dc0f64ab2b6a29b2dc4b84875: Log is configured to *not* fsync() on all Append() calls
22:18:18.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.878671 22900 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f65d617dc0f64ab2b6a29b2dc4b84875: No bootstrap required, opened a new log
22:18:18.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.898792 22900 raft_consensus.cc:348] T 00000000000000000000000000000000 P f65d617dc0f64ab2b6a29b2dc4b84875 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f65d617dc0f64ab2b6a29b2dc4b84875" member_type: VOTER last_known_addr { host: "127.22.73.62" port: 42139 } } peers { permanent_uuid: "a5ed30b572d44c96a73bcc696ea0282f" member_type: VOTER last_known_addr { host: "127.22.73.61" port: 36531 } } peers { permanent_uuid: "2d5ca51e59ac4b5f97da5e6c0d50f839" member_type: VOTER last_known_addr { host: "127.22.73.60" port: 38817 } }
22:18:18.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.899411 22900 raft_consensus.cc:374] T 00000000000000000000000000000000 P f65d617dc0f64ab2b6a29b2dc4b84875 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
22:18:18.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.899688 22900 raft_consensus.cc:729] T 00000000000000000000000000000000 P f65d617dc0f64ab2b6a29b2dc4b84875 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f65d617dc0f64ab2b6a29b2dc4b84875, State: Initialized, Role: FOLLOWER
22:18:18.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.900462 22900 consensus_queue.cc:260] T 00000000000000000000000000000000 P f65d617dc0f64ab2b6a29b2dc4b84875 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f65d617dc0f64ab2b6a29b2dc4b84875" member_type: VOTER last_known_addr { host: "127.22.73.62" port: 42139 } } peers { permanent_uuid: "a5ed30b572d44c96a73bcc696ea0282f" member_type: VOTER last_known_addr { host: "127.22.73.61" port: 36531 } } peers { permanent_uuid: "2d5ca51e59ac4b5f97da5e6c0d50f839" member_type: VOTER last_known_addr { host: "127.22.73.60" port: 38817 } }
22:18:18.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.903447 23078 sys_catalog.cc:455] T 00000000000000000000000000000000 P f65d617dc0f64ab2b6a29b2dc4b84875 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f65d617dc0f64ab2b6a29b2dc4b84875" member_type: VOTER last_known_addr { host: "127.22.73.62" port: 42139 } } peers { permanent_uuid: "a5ed30b572d44c96a73bcc696ea0282f" member_type: VOTER last_known_addr { host: "127.22.73.61" port: 36531 } } peers { permanent_uuid: "2d5ca51e59ac4b5f97da5e6c0d50f839" member_type: VOTER last_known_addr { host: "127.22.73.60" port: 38817 } } }
22:18:18.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.904392 23078 sys_catalog.cc:458] T 00000000000000000000000000000000 P f65d617dc0f64ab2b6a29b2dc4b84875 [sys.catalog]: This master's current role is: FOLLOWER
22:18:18.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.905520 22900 sys_catalog.cc:564] T 00000000000000000000000000000000 P f65d617dc0f64ab2b6a29b2dc4b84875 [sys.catalog]: configured and running, proceeding with master startup.
22:18:18.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:18.936000 23089 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f65d617dc0f64ab2b6a29b2dc4b84875: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:18:18.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:18.936347 23089 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:18:18.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.940929 23078 raft_consensus.cc:482] T 00000000000000000000000000000000 P f65d617dc0f64ab2b6a29b2dc4b84875 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
22:18:18.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.941375 23078 raft_consensus.cc:504] T 00000000000000000000000000000000 P f65d617dc0f64ab2b6a29b2dc4b84875 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f65d617dc0f64ab2b6a29b2dc4b84875" member_type: VOTER last_known_addr { host: "127.22.73.62" port: 42139 } } peers { permanent_uuid: "a5ed30b572d44c96a73bcc696ea0282f" member_type: VOTER last_known_addr { host: "127.22.73.61" port: 36531 } } peers { permanent_uuid: "2d5ca51e59ac4b5f97da5e6c0d50f839" member_type: VOTER last_known_addr { host: "127.22.73.60" port: 38817 } }
22:18:18.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.946161 23029 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f65d617dc0f64ab2b6a29b2dc4b84875" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2d5ca51e59ac4b5f97da5e6c0d50f839" is_pre_election: true
22:18:18.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.947001 23029 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 2d5ca51e59ac4b5f97da5e6c0d50f839 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f65d617dc0f64ab2b6a29b2dc4b84875 in term 0.
22:18:18.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.949012 22849 leader_election.cc:302] T 00000000000000000000000000000000 P f65d617dc0f64ab2b6a29b2dc4b84875 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2d5ca51e59ac4b5f97da5e6c0d50f839, f65d617dc0f64ab2b6a29b2dc4b84875; no voters:
22:18:18.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.952137 23078 leader_election.cc:288] T 00000000000000000000000000000000 P f65d617dc0f64ab2b6a29b2dc4b84875 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a5ed30b572d44c96a73bcc696ea0282f (127.22.73.61:36531), 2d5ca51e59ac4b5f97da5e6c0d50f839 (127.22.73.60:38817)
22:18:18.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.952782 23078 raft_consensus.cc:2781] T 00000000000000000000000000000000 P f65d617dc0f64ab2b6a29b2dc4b84875 [term 0 FOLLOWER]: Leader pre-election won for term 1
22:18:18.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.953083 23078 raft_consensus.cc:482] T 00000000000000000000000000000000 P f65d617dc0f64ab2b6a29b2dc4b84875 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
22:18:18.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.953390 23078 raft_consensus.cc:3037] T 00000000000000000000000000000000 P f65d617dc0f64ab2b6a29b2dc4b84875 [term 0 FOLLOWER]: Advancing to term 1
22:18:18.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:18.955835 22854 tablet.cc:2369] T 00000000000000000000000000000000 P f65d617dc0f64ab2b6a29b2dc4b84875: Can't schedule compaction. Clean time has not been advanced past its initial value.
22:18:18.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.932127 22832 master_runner.cc:186] Error getting master registration for 127.22.73.61:36531: OK, instance_id { permanent_uuid: "a5ed30b572d44c96a73bcc696ea0282f" instance_seqno: 1790547496960202 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
22:18:18.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.968717 22832 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
22:18:18.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.968971 23078 raft_consensus.cc:504] T 00000000000000000000000000000000 P f65d617dc0f64ab2b6a29b2dc4b84875 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f65d617dc0f64ab2b6a29b2dc4b84875" member_type: VOTER last_known_addr { host: "127.22.73.62" port: 42139 } } peers { permanent_uuid: "a5ed30b572d44c96a73bcc696ea0282f" member_type: VOTER last_known_addr { host: "127.22.73.61" port: 36531 } } peers { permanent_uuid: "2d5ca51e59ac4b5f97da5e6c0d50f839" member_type: VOTER last_known_addr { host: "127.22.73.60" port: 38817 } }
22:18:18.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.971841 23078 leader_election.cc:288] T 00000000000000000000000000000000 P f65d617dc0f64ab2b6a29b2dc4b84875 [CANDIDATE]: Term 1 election: Requested vote from peers a5ed30b572d44c96a73bcc696ea0282f (127.22.73.61:36531), 2d5ca51e59ac4b5f97da5e6c0d50f839 (127.22.73.60:38817)
22:18:18.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.977411 23029 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f65d617dc0f64ab2b6a29b2dc4b84875" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2d5ca51e59ac4b5f97da5e6c0d50f839"
22:18:18.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.978016 23029 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 2d5ca51e59ac4b5f97da5e6c0d50f839 [term 0 FOLLOWER]: Advancing to term 1
22:18:18.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.986176 23029 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 2d5ca51e59ac4b5f97da5e6c0d50f839 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f65d617dc0f64ab2b6a29b2dc4b84875 in term 1.
22:18:18.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.987967 22849 leader_election.cc:302] T 00000000000000000000000000000000 P f65d617dc0f64ab2b6a29b2dc4b84875 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2d5ca51e59ac4b5f97da5e6c0d50f839, f65d617dc0f64ab2b6a29b2dc4b84875; no voters:
22:18:18.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.988883 23078 raft_consensus.cc:2781] T 00000000000000000000000000000000 P f65d617dc0f64ab2b6a29b2dc4b84875 [term 1 FOLLOWER]: Leader election won for term 1
22:18:18.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.989451 23078 raft_consensus.cc:686] T 00000000000000000000000000000000 P f65d617dc0f64ab2b6a29b2dc4b84875 [term 1 LEADER]: Becoming Leader. State: Replica: f65d617dc0f64ab2b6a29b2dc4b84875, State: Running, Role: LEADER
22:18:18.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:18.990648 23078 consensus_queue.cc:237] T 00000000000000000000000000000000 P f65d617dc0f64ab2b6a29b2dc4b84875 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f65d617dc0f64ab2b6a29b2dc4b84875" member_type: VOTER last_known_addr { host: "127.22.73.62" port: 42139 } } peers { permanent_uuid: "a5ed30b572d44c96a73bcc696ea0282f" member_type: VOTER last_known_addr { host: "127.22.73.61" port: 36531 } } peers { permanent_uuid: "2d5ca51e59ac4b5f97da5e6c0d50f839" member_type: VOTER last_known_addr { host: "127.22.73.60" port: 38817 } }
22:18:19.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.012703 23091 sys_catalog.cc:455] T 00000000000000000000000000000000 P f65d617dc0f64ab2b6a29b2dc4b84875 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f65d617dc0f64ab2b6a29b2dc4b84875. Latest consensus state: current_term: 1 leader_uuid: "f65d617dc0f64ab2b6a29b2dc4b84875" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f65d617dc0f64ab2b6a29b2dc4b84875" member_type: VOTER last_known_addr { host: "127.22.73.62" port: 42139 } } peers { permanent_uuid: "a5ed30b572d44c96a73bcc696ea0282f" member_type: VOTER last_known_addr { host: "127.22.73.61" port: 36531 } } peers { permanent_uuid: "2d5ca51e59ac4b5f97da5e6c0d50f839" member_type: VOTER last_known_addr { host: "127.22.73.60" port: 38817 } } }
22:18:19.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.014526 23091 sys_catalog.cc:458] T 00000000000000000000000000000000 P f65d617dc0f64ab2b6a29b2dc4b84875 [sys.catalog]: This master's current role is: LEADER
22:18:19.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.025018 23093 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
22:18:19.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.031970 23093 catalog_manager.cc:1484] Initializing Kudu cluster ID...
22:18:19.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.053563 23029 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 2d5ca51e59ac4b5f97da5e6c0d50f839 [term 1 FOLLOWER]: Refusing update from remote peer f65d617dc0f64ab2b6a29b2dc4b84875: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
22:18:19.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.055183 23078 consensus_queue.cc:1035] T 00000000000000000000000000000000 P f65d617dc0f64ab2b6a29b2dc4b84875 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2d5ca51e59ac4b5f97da5e6c0d50f839" member_type: VOTER last_known_addr { host: "127.22.73.60" port: 38817 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
22:18:19.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.085156 23063 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2d5ca51e59ac4b5f97da5e6c0d50f839 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f65d617dc0f64ab2b6a29b2dc4b84875. Latest consensus state: current_term: 1 leader_uuid: "f65d617dc0f64ab2b6a29b2dc4b84875" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f65d617dc0f64ab2b6a29b2dc4b84875" member_type: VOTER last_known_addr { host: "127.22.73.62" port: 42139 } } peers { permanent_uuid: "a5ed30b572d44c96a73bcc696ea0282f" member_type: VOTER last_known_addr { host: "127.22.73.61" port: 36531 } } peers { permanent_uuid: "2d5ca51e59ac4b5f97da5e6c0d50f839" member_type: VOTER last_known_addr { host: "127.22.73.60" port: 38817 } } }
22:18:19.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.085868 23063 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2d5ca51e59ac4b5f97da5e6c0d50f839 [sys.catalog]: This master's current role is: FOLLOWER
22:18:19.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.104926 23078 sys_catalog.cc:455] T 00000000000000000000000000000000 P f65d617dc0f64ab2b6a29b2dc4b84875 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "f65d617dc0f64ab2b6a29b2dc4b84875" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f65d617dc0f64ab2b6a29b2dc4b84875" member_type: VOTER last_known_addr { host: "127.22.73.62" port: 42139 } } peers { permanent_uuid: "a5ed30b572d44c96a73bcc696ea0282f" member_type: VOTER last_known_addr { host: "127.22.73.61" port: 36531 } } peers { permanent_uuid: "2d5ca51e59ac4b5f97da5e6c0d50f839" member_type: VOTER last_known_addr { host: "127.22.73.60" port: 38817 } } }
22:18:19.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.105645 23078 sys_catalog.cc:458] T 00000000000000000000000000000000 P f65d617dc0f64ab2b6a29b2dc4b84875 [sys.catalog]: This master's current role is: LEADER
22:18:19.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.116212 23063 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2d5ca51e59ac4b5f97da5e6c0d50f839 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "f65d617dc0f64ab2b6a29b2dc4b84875" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f65d617dc0f64ab2b6a29b2dc4b84875" member_type: VOTER last_known_addr { host: "127.22.73.62" port: 42139 } } peers { permanent_uuid: "a5ed30b572d44c96a73bcc696ea0282f" member_type: VOTER last_known_addr { host: "127.22.73.61" port: 36531 } } peers { permanent_uuid: "2d5ca51e59ac4b5f97da5e6c0d50f839" member_type: VOTER last_known_addr { host: "127.22.73.60" port: 38817 } } }
22:18:19.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.116894 23063 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2d5ca51e59ac4b5f97da5e6c0d50f839 [sys.catalog]: This master's current role is: FOLLOWER
22:18:19.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.118464 23093 catalog_manager.cc:1347] Generated new cluster ID: 1d627e731b674117a59c0d370f09c91b
22:18:19.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.118721 23093 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
22:18:19.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.170789 23093 catalog_manager.cc:1370] Generated new certificate authority record
22:18:19.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.173518 23093 catalog_manager.cc:1504] Loading token signing keys...
22:18:19.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.217162 23093 catalog_manager.cc:5868] T 00000000000000000000000000000000 P f65d617dc0f64ab2b6a29b2dc4b84875: Generated new TSK 0
22:18:19.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.218438 23093 catalog_manager.cc:1514] Initializing in-progress tserver states...
22:18:19.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.259215 22820 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:18:19.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:18:19.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-0/wal
22:18:19.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-0/data
22:18:19.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
22:18:19.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
22:18:19.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
22:18:19.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:18:19.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:18:19.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:18:19.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-0/logs
22:18:19.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-0/data/info.pb
22:18:19.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:18:19.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
22:18:19.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
22:18:19.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
22:18:19.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
22:18:19.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:18:19.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:18:19.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:18:19.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
22:18:19.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
22:18:19.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.73.1:0
22:18:19.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.73.1
22:18:19.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.73.1
22:18:19.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:18:19.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.73.62:42139,127.22.73.61:36531,127.22.73.60:38817
22:18:19.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.73.20:34075
22:18:19.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:18:19.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:18:19.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
22:18:19.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.324741 22975 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.73.60" port: 38817 } attempt: 7
22:18:19.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.385219 22975 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a5ed30b572d44c96a73bcc696ea0282f: Bootstrap starting.
22:18:19.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.402446 22975 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a5ed30b572d44c96a73bcc696ea0282f: Neither blocks nor log segments found. Creating new log.
22:18:19.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.404763 22975 log.cc:824] T 00000000000000000000000000000000 P a5ed30b572d44c96a73bcc696ea0282f: Log is configured to *not* fsync() on all Append() calls
22:18:19.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.428050 22975 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a5ed30b572d44c96a73bcc696ea0282f: No bootstrap required, opened a new log
22:18:19.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.485447 22975 raft_consensus.cc:348] T 00000000000000000000000000000000 P a5ed30b572d44c96a73bcc696ea0282f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f65d617dc0f64ab2b6a29b2dc4b84875" member_type: VOTER last_known_addr { host: "127.22.73.62" port: 42139 } } peers { permanent_uuid: "a5ed30b572d44c96a73bcc696ea0282f" member_type: VOTER last_known_addr { host: "127.22.73.61" port: 36531 } } peers { permanent_uuid: "2d5ca51e59ac4b5f97da5e6c0d50f839" member_type: VOTER last_known_addr { host: "127.22.73.60" port: 38817 } }
22:18:19.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.490682 22975 raft_consensus.cc:374] T 00000000000000000000000000000000 P a5ed30b572d44c96a73bcc696ea0282f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
22:18:19.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.491091 22975 raft_consensus.cc:729] T 00000000000000000000000000000000 P a5ed30b572d44c96a73bcc696ea0282f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a5ed30b572d44c96a73bcc696ea0282f, State: Initialized, Role: FOLLOWER
22:18:19.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.492089 22975 consensus_queue.cc:260] T 00000000000000000000000000000000 P a5ed30b572d44c96a73bcc696ea0282f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f65d617dc0f64ab2b6a29b2dc4b84875" member_type: VOTER last_known_addr { host: "127.22.73.62" port: 42139 } } peers { permanent_uuid: "a5ed30b572d44c96a73bcc696ea0282f" member_type: VOTER last_known_addr { host: "127.22.73.61" port: 36531 } } peers { permanent_uuid: "2d5ca51e59ac4b5f97da5e6c0d50f839" member_type: VOTER last_known_addr { host: "127.22.73.60" port: 38817 } }
22:18:19.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.495877 23103 sys_catalog.cc:455] T 00000000000000000000000000000000 P a5ed30b572d44c96a73bcc696ea0282f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f65d617dc0f64ab2b6a29b2dc4b84875" member_type: VOTER last_known_addr { host: "127.22.73.62" port: 42139 } } peers { permanent_uuid: "a5ed30b572d44c96a73bcc696ea0282f" member_type: VOTER last_known_addr { host: "127.22.73.61" port: 36531 } } peers { permanent_uuid: "2d5ca51e59ac4b5f97da5e6c0d50f839" member_type: VOTER last_known_addr { host: "127.22.73.60" port: 38817 } } }
22:18:19.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.497797 23103 sys_catalog.cc:458] T 00000000000000000000000000000000 P a5ed30b572d44c96a73bcc696ea0282f [sys.catalog]: This master's current role is: FOLLOWER
22:18:19.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.500873 22975 sys_catalog.cc:564] T 00000000000000000000000000000000 P a5ed30b572d44c96a73bcc696ea0282f [sys.catalog]: configured and running, proceeding with master startup.
22:18:19.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.501042 22950 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f65d617dc0f64ab2b6a29b2dc4b84875" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a5ed30b572d44c96a73bcc696ea0282f" is_pre_election: true
22:18:19.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.501942 22950 raft_consensus.cc:2454] T 00000000000000000000000000000000 P a5ed30b572d44c96a73bcc696ea0282f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f65d617dc0f64ab2b6a29b2dc4b84875 in term 0.
22:18:19.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.504427 22949 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f65d617dc0f64ab2b6a29b2dc4b84875" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a5ed30b572d44c96a73bcc696ea0282f"
22:18:19.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.505491 22948 raft_consensus.cc:3037] T 00000000000000000000000000000000 P a5ed30b572d44c96a73bcc696ea0282f [term 0 FOLLOWER]: Advancing to term 1
22:18:19.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.513689 22948 raft_consensus.cc:1261] T 00000000000000000000000000000000 P a5ed30b572d44c96a73bcc696ea0282f [term 1 FOLLOWER]: Refusing update from remote peer f65d617dc0f64ab2b6a29b2dc4b84875: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
22:18:19.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.515261 23099 consensus_queue.cc:1035] T 00000000000000000000000000000000 P f65d617dc0f64ab2b6a29b2dc4b84875 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a5ed30b572d44c96a73bcc696ea0282f" member_type: VOTER last_known_addr { host: "127.22.73.61" port: 36531 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
22:18:19.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.588933 23103 sys_catalog.cc:455] T 00000000000000000000000000000000 P a5ed30b572d44c96a73bcc696ea0282f [sys.catalog]: SysCatalogTable state changed. Reason: New leader f65d617dc0f64ab2b6a29b2dc4b84875. Latest consensus state: current_term: 1 leader_uuid: "f65d617dc0f64ab2b6a29b2dc4b84875" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f65d617dc0f64ab2b6a29b2dc4b84875" member_type: VOTER last_known_addr { host: "127.22.73.62" port: 42139 } } peers { permanent_uuid: "a5ed30b572d44c96a73bcc696ea0282f" member_type: VOTER last_known_addr { host: "127.22.73.61" port: 36531 } } peers { permanent_uuid: "2d5ca51e59ac4b5f97da5e6c0d50f839" member_type: VOTER last_known_addr { host: "127.22.73.60" port: 38817 } } }
22:18:19.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.589097 23111 sys_catalog.cc:455] T 00000000000000000000000000000000 P a5ed30b572d44c96a73bcc696ea0282f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "f65d617dc0f64ab2b6a29b2dc4b84875" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f65d617dc0f64ab2b6a29b2dc4b84875" member_type: VOTER last_known_addr { host: "127.22.73.62" port: 42139 } } peers { permanent_uuid: "a5ed30b572d44c96a73bcc696ea0282f" member_type: VOTER last_known_addr { host: "127.22.73.61" port: 36531 } } peers { permanent_uuid: "2d5ca51e59ac4b5f97da5e6c0d50f839" member_type: VOTER last_known_addr { host: "127.22.73.60" port: 38817 } } }
22:18:19.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.589998 23103 sys_catalog.cc:458] T 00000000000000000000000000000000 P a5ed30b572d44c96a73bcc696ea0282f [sys.catalog]: This master's current role is: FOLLOWER
22:18:19.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.590009 23111 sys_catalog.cc:458] T 00000000000000000000000000000000 P a5ed30b572d44c96a73bcc696ea0282f [sys.catalog]: This master's current role is: FOLLOWER
22:18:19.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.625933 23119 catalog_manager.cc:1259] Loaded cluster ID: 1d627e731b674117a59c0d370f09c91b
22:18:19.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.626364 23119 catalog_manager.cc:1552] T 00000000000000000000000000000000 P a5ed30b572d44c96a73bcc696ea0282f: loading cluster ID for follower catalog manager: success
22:18:19.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.633801 23119 catalog_manager.cc:1574] T 00000000000000000000000000000000 P a5ed30b572d44c96a73bcc696ea0282f: acquiring CA information for follower catalog manager: success
22:18:19.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.639279 23119 catalog_manager.cc:1602] T 00000000000000000000000000000000 P a5ed30b572d44c96a73bcc696ea0282f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
22:18:19.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:19.764832 23100 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
22:18:19.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:19.765648 23100 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
22:18:19.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:19.766538 23100 flags.cc:425] Enabled unsafe flag: --never_fsync=true
22:18:19.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.808439 23074 catalog_manager.cc:1259] Loaded cluster ID: 1d627e731b674117a59c0d370f09c91b
22:18:19.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.808830 23074 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 2d5ca51e59ac4b5f97da5e6c0d50f839: loading cluster ID for follower catalog manager: success
22:18:19.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:19.820156 23100 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
22:18:19.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:19.821643 23100 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.73.1
22:18:19.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.829054 23074 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 2d5ca51e59ac4b5f97da5e6c0d50f839: acquiring CA information for follower catalog manager: success
22:18:19.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.837090 23074 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 2d5ca51e59ac4b5f97da5e6c0d50f839: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
22:18:19.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.880133 23100 tablet_server_runner.cc:78] Tablet server non-default flags:
22:18:19.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:18:19.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.73.20:34075
22:18:19.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:18:19.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:18:19.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-0/data
22:18:19.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-0/wal
22:18:19.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:18:19.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:18:19.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.73.1:0
22:18:19.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
22:18:19.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:18:19.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:18:19.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-0/data/info.pb
22:18:19.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.73.1
22:18:19.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:18:19.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.73.62:42139,127.22.73.61:36531,127.22.73.60:38817
22:18:19.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
22:18:19.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:18:19.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
22:18:19.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
22:18:19.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:18:19.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.73.1
22:18:19.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-0/logs
22:18:19.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:18:19.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
22:18:19.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:18:19.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
22:18:19.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
22:18:19.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
22:18:19.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
22:18:19.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 21:22:24 UTC on ccycloud.quasar-wysbne.root.comops.site
22:18:19.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676318
22:18:19.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
22:18:19.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.883307 23100 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
22:18:19.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.885479 23100 file_cache.cc:493] Constructed file cache file cache with capacity 419430
22:18:19.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:19.924292 23127 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:18:19.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.997416 23099 sys_catalog.cc:455] T 00000000000000000000000000000000 P f65d617dc0f64ab2b6a29b2dc4b84875 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "f65d617dc0f64ab2b6a29b2dc4b84875" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f65d617dc0f64ab2b6a29b2dc4b84875" member_type: VOTER last_known_addr { host: "127.22.73.62" port: 42139 } } peers { permanent_uuid: "a5ed30b572d44c96a73bcc696ea0282f" member_type: VOTER last_known_addr { host: "127.22.73.61" port: 36531 } } peers { permanent_uuid: "2d5ca51e59ac4b5f97da5e6c0d50f839" member_type: VOTER last_known_addr { host: "127.22.73.60" port: 38817 } } }
22:18:19.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:19.998302 23099 sys_catalog.cc:458] T 00000000000000000000000000000000 P f65d617dc0f64ab2b6a29b2dc4b84875 [sys.catalog]: This master's current role is: LEADER
22:18:22.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:21.306734 23125 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 23100
22:18:22.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:22.158923 23100 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 2.231s user 0.583s sys 0.911s
22:18:22.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:22.161273 23130 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:18:22.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:22.161497 23100 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 2.233s user 0.583s sys 0.911s
22:18:22.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:19.934435 23126 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:18:22.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.162038 23100 server_base.cc:1034] running on GCE node
22:18:22.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.163460 23100 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
22:18:22.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.166399 23100 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
22:18:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.167806 23100 hybrid_clock.cc:648] HybridClock initialized: now 1790547502167749 us; error 67 us; skew 500 ppm
22:18:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.168663 23100 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
22:18:22.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.181762 23100 webserver.cc:460] Webserver started at http://127.22.73.1:33059/ using document root <none> and password file <none>
22:18:22.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.182796 23100 fs_manager.cc:359] Metadata directory not provided
22:18:22.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.183058 23100 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:18:22.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.183596 23100 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
22:18:22.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.188345 23100 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-0/data/instance:
22:18:22.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3af1027bdbe546199b72b17ccb10c361"
22:18:22.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:18:22 on dist-test-slave-qzth"
22:18:22.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.189544 23100 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-0/wal/instance:
22:18:22.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3af1027bdbe546199b72b17ccb10c361"
22:18:22.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:18:22 on dist-test-slave-qzth"
22:18:22.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.197031 23100 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.001s sys 0.005s
22:18:22.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.203104 23138 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
22:18:22.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.204363 23100 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
22:18:22.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.204735 23100 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-0/data,/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-0/wal
22:18:22.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3af1027bdbe546199b72b17ccb10c361"
22:18:22.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:18:22 on dist-test-slave-qzth"
22:18:22.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.205103 23100 fs_report.cc:383] FS layout report
22:18:22.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
22:18:22.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-0/wal
22:18:22.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-0/wal
22:18:22.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-0/data/data
22:18:22.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
22:18:22.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
22:18:22.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
22:18:22.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
22:18:22.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
22:18:22.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
22:18:22.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
22:18:22.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
22:18:22.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
22:18:22.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
22:18:22.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
22:18:22.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.265799 23100 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
22:18:22.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.267795 23100 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
22:18:22.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.268314 23100 kserver.cc:163] Server-wide thread pool size limit: 3276
22:18:22.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.279407 23100 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
22:18:22.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.285115 23100 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
22:18:22.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.285389 23100 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
22:18:22.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.285671 23100 ts_tablet_manager.cc:610] Registered 0 tablets
22:18:22.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.285889 23100 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.001s sys 0.000s
22:18:22.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.471963 23100 rpc_server.cc:307] RPC server started. Bound to: 127.22.73.1:35415
22:18:22.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.472086 23250 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.73.1:35415 every 8 connection(s)
22:18:22.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.475230 23100 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-0/data/info.pb
22:18:22.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.480039 22820 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu as pid 23100
22:18:22.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.480602 22820 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-0/wal/instance
22:18:22.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.491993 22820 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:18:22.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:18:22.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-1/wal
22:18:22.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-1/data
22:18:22.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
22:18:22.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
22:18:22.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
22:18:22.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:18:22.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:18:22.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:18:22.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-1/logs
22:18:22.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-1/data/info.pb
22:18:22.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:18:22.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
22:18:22.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
22:18:22.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
22:18:22.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
22:18:22.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:18:22.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:18:22.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:18:22.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
22:18:22.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
22:18:22.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.73.2:0
22:18:22.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.73.2
22:18:22.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.73.2
22:18:22.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:18:22.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.73.62:42139,127.22.73.61:36531,127.22.73.60:38817
22:18:22.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.73.20:34075
22:18:22.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:18:22.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:18:22.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
22:18:22.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.546239 23252 heartbeater.cc:347] Connected to a master server at 127.22.73.61:36531
22:18:22.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.546691 23252 heartbeater.cc:464] Registering TS with master...
22:18:22.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.547811 23252 heartbeater.cc:511] Master 127.22.73.61:36531 requested a full tablet report, sending...
22:18:22.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.551231 22940 ts_manager.cc:194] Registered new tserver with Master: 3af1027bdbe546199b72b17ccb10c361 (127.22.73.1:35415)
22:18:22.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.554385 23255 heartbeater.cc:347] Connected to a master server at 127.22.73.62:42139
22:18:22.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.554703 23255 heartbeater.cc:464] Registering TS with master...
22:18:22.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.555437 23255 heartbeater.cc:511] Master 127.22.73.62:42139 requested a full tablet report, sending...
22:18:22.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.557709 22864 ts_manager.cc:194] Registered new tserver with Master: 3af1027bdbe546199b72b17ccb10c361 (127.22.73.1:35415)
22:18:22.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.560586 22864 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.22.73.1:48701
22:18:22.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.559209 23251 heartbeater.cc:347] Connected to a master server at 127.22.73.60:38817
22:18:22.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.564664 23251 heartbeater.cc:464] Registering TS with master...
22:18:22.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.565480 23251 heartbeater.cc:511] Master 127.22.73.60:38817 requested a full tablet report, sending...
22:18:22.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.568154 23019 ts_manager.cc:194] Registered new tserver with Master: 3af1027bdbe546199b72b17ccb10c361 (127.22.73.1:35415)
22:18:22.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:22.876930 23258 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
22:18:22.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:22.877354 23258 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
22:18:22.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:22.877753 23258 flags.cc:425] Enabled unsafe flag: --never_fsync=true
22:18:22.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:22.907913 23258 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
22:18:22.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:22.908664 23258 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.73.2
22:18:22.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.940644 23258 tablet_server_runner.cc:78] Tablet server non-default flags:
22:18:22.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:18:22.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.73.20:34075
22:18:22.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:18:22.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:18:22.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-1/data
22:18:22.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-1/wal
22:18:22.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:18:22.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:18:22.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.73.2:0
22:18:22.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
22:18:22.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:18:22.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:18:22.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-1/data/info.pb
22:18:22.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.73.2
22:18:22.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:18:22.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.73.62:42139,127.22.73.61:36531,127.22.73.60:38817
22:18:22.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
22:18:22.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:18:22.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
22:18:22.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
22:18:22.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:18:22.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.73.2
22:18:22.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-1/logs
22:18:22.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:18:22.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
22:18:22.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:18:22.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
22:18:22.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
22:18:22.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
22:18:22.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
22:18:22.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 21:22:24 UTC on ccycloud.quasar-wysbne.root.comops.site
22:18:22.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676318
22:18:22.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
22:18:22.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.941707 23258 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
22:18:22.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.943375 23258 file_cache.cc:493] Constructed file cache file cache with capacity 419430
22:18:22.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:22.961022 23268 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:18:23.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:23.572075 23255 heartbeater.cc:503] Master 127.22.73.62:42139 was elected leader, sending a full tablet report...
22:18:24.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:22.961217 23267 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:18:24.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:22.961777 23270 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:18:24.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:22.964068 23258 server_base.cc:1034] running on GCE node
22:18:24.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:24.155052 23258 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
22:18:24.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:24.157604 23258 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
22:18:24.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:24.159034 23258 hybrid_clock.cc:648] HybridClock initialized: now 1790547504158971 us; error 69 us; skew 500 ppm
22:18:24.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:24.159847 23258 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
22:18:24.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:24.166754 23258 webserver.cc:460] Webserver started at http://127.22.73.2:35867/ using document root <none> and password file <none>
22:18:24.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:24.167570 23258 fs_manager.cc:359] Metadata directory not provided
22:18:24.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:24.167765 23258 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:18:24.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:24.168183 23258 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
22:18:24.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:24.172945 23258 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-1/data/instance:
22:18:24.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e9774e54f536435589fa7993d282759c"
22:18:24.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:18:24 on dist-test-slave-qzth"
22:18:24.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:24.174348 23258 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-1/wal/instance:
22:18:24.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e9774e54f536435589fa7993d282759c"
22:18:24.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:18:24 on dist-test-slave-qzth"
22:18:24.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:24.181515 23258 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.006s sys 0.001s
22:18:24.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:24.187053 23280 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
22:18:24.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:24.188083 23258 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.002s
22:18:24.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:24.188366 23258 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-1/data,/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-1/wal
22:18:24.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e9774e54f536435589fa7993d282759c"
22:18:24.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:18:24 on dist-test-slave-qzth"
22:18:24.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:24.188652 23258 fs_report.cc:383] FS layout report
22:18:24.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
22:18:24.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-1/wal
22:18:24.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-1/wal
22:18:24.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-1/data/data
22:18:24.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
22:18:24.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
22:18:24.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
22:18:24.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
22:18:24.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
22:18:24.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
22:18:24.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
22:18:24.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
22:18:24.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
22:18:24.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
22:18:24.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
22:18:24.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:24.227468 23258 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
22:18:24.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:24.228922 23258 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
22:18:24.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:24.229348 23258 kserver.cc:163] Server-wide thread pool size limit: 3276
22:18:24.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:24.231848 23258 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
22:18:24.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:24.235599 23258 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
22:18:24.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:24.235775 23258 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
22:18:24.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:24.235970 23258 ts_tablet_manager.cc:610] Registered 0 tablets
22:18:24.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:24.236089 23258 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
22:18:24.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:24.376969 23258 rpc_server.cc:307] RPC server started. Bound to: 127.22.73.2:40829
22:18:24.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:24.377085 23392 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.73.2:40829 every 8 connection(s)
22:18:24.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:24.379376 23258 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-1/data/info.pb
22:18:24.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:24.382419 22820 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu as pid 23258
22:18:24.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:24.383080 22820 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-1/wal/instance
22:18:24.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:24.404040 22820 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:18:24.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:18:24.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-2/wal
22:18:24.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-2/data
22:18:24.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
22:18:24.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
22:18:24.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
22:18:24.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:18:24.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:18:24.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:18:24.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-2/logs
22:18:24.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-2/data/info.pb
22:18:24.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:18:24.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
22:18:24.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
22:18:24.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
22:18:24.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
22:18:24.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:18:24.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:18:24.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:18:24.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
22:18:24.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
22:18:24.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.73.3:0
22:18:24.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.73.3
22:18:24.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.73.3
22:18:24.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:18:24.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.73.62:42139,127.22.73.61:36531,127.22.73.60:38817
22:18:24.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.73.20:34075
22:18:24.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:18:24.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:18:24.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
22:18:24.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:24.440661 23393 heartbeater.cc:347] Connected to a master server at 127.22.73.60:38817
22:18:24.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:24.441181 23393 heartbeater.cc:464] Registering TS with master...
22:18:24.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:24.442313 23394 heartbeater.cc:347] Connected to a master server at 127.22.73.61:36531
22:18:24.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:24.442653 23394 heartbeater.cc:464] Registering TS with master...
22:18:24.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:24.443763 23393 heartbeater.cc:511] Master 127.22.73.60:38817 requested a full tablet report, sending...
22:18:24.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:24.443845 23394 heartbeater.cc:511] Master 127.22.73.61:36531 requested a full tablet report, sending...
22:18:24.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:24.446807 23019 ts_manager.cc:194] Registered new tserver with Master: e9774e54f536435589fa7993d282759c (127.22.73.2:40829)
22:18:24.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:24.446816 22940 ts_manager.cc:194] Registered new tserver with Master: e9774e54f536435589fa7993d282759c (127.22.73.2:40829)
22:18:24.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:24.447170 23395 heartbeater.cc:347] Connected to a master server at 127.22.73.62:42139
22:18:24.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:24.447476 23395 heartbeater.cc:464] Registering TS with master...
22:18:24.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:24.448256 23395 heartbeater.cc:511] Master 127.22.73.62:42139 requested a full tablet report, sending...
22:18:24.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:24.450383 22864 ts_manager.cc:194] Registered new tserver with Master: e9774e54f536435589fa7993d282759c (127.22.73.2:40829)
22:18:24.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:24.452471 22864 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.22.73.2:46967
22:18:24.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:24.729919 23403 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
22:18:24.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:24.730352 23403 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
22:18:24.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:24.730773 23403 flags.cc:425] Enabled unsafe flag: --never_fsync=true
22:18:24.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:24.760923 23403 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
22:18:24.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:24.761711 23403 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.73.3
22:18:24.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:24.796265 23403 tablet_server_runner.cc:78] Tablet server non-default flags:
22:18:24.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:18:24.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.73.20:34075
22:18:24.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:18:24.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:18:24.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-2/data
22:18:24.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-2/wal
22:18:24.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:18:24.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:18:24.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.73.3:0
22:18:24.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
22:18:24.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:18:24.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:18:24.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-2/data/info.pb
22:18:24.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.73.3
22:18:24.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:18:24.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.73.62:42139,127.22.73.61:36531,127.22.73.60:38817
22:18:24.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
22:18:24.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:18:24.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
22:18:24.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
22:18:24.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:18:24.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.73.3
22:18:24.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-2/logs
22:18:24.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:18:24.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
22:18:24.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:18:24.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
22:18:24.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
22:18:24.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
22:18:24.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
22:18:24.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 21:22:24 UTC on ccycloud.quasar-wysbne.root.comops.site
22:18:24.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676318
22:18:24.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
22:18:24.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:24.797339 23403 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
22:18:24.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:24.798894 23403 file_cache.cc:493] Constructed file cache file cache with capacity 419430
22:18:24.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:24.817103 23412 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:18:25.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:25.456503 23395 heartbeater.cc:503] Master 127.22.73.62:42139 was elected leader, sending a full tablet report...
22:18:25.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:24.817157 23410 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:18:25.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:24.817186 23409 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:18:25.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:24.818895 23403 server_base.cc:1034] running on GCE node
22:18:25.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:25.995393 23403 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
22:18:25.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:25.998135 23403 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
22:18:25.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:25.999575 23403 hybrid_clock.cc:648] HybridClock initialized: now 1790547505999542 us; error 43 us; skew 500 ppm
22:18:26.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:26.000376 23403 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
22:18:26.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:26.011449 23403 webserver.cc:460] Webserver started at http://127.22.73.3:34983/ using document root <none> and password file <none>
22:18:26.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:26.012522 23403 fs_manager.cc:359] Metadata directory not provided
22:18:26.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:26.012765 23403 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:18:26.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:26.013147 23403 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
22:18:26.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:26.018278 23403 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-2/data/instance:
22:18:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aecd5d2d8771486fb4265da8de3b5f34"
22:18:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:18:26 on dist-test-slave-qzth"
22:18:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:26.019304 23403 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-2/wal/instance:
22:18:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aecd5d2d8771486fb4265da8de3b5f34"
22:18:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:18:26 on dist-test-slave-qzth"
22:18:26.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:26.026839 23403 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
22:18:26.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:26.032425 23420 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
22:18:26.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:26.033454 23403 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
22:18:26.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:26.033715 23403 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-2/data,/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-2/wal
22:18:26.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aecd5d2d8771486fb4265da8de3b5f34"
22:18:26.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:18:26 on dist-test-slave-qzth"
22:18:26.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:26.034080 23403 fs_report.cc:383] FS layout report
22:18:26.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
22:18:26.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-2/wal
22:18:26.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-2/wal
22:18:26.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-2/data/data
22:18:26.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
22:18:26.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
22:18:26.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
22:18:26.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
22:18:26.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
22:18:26.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
22:18:26.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
22:18:26.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
22:18:26.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
22:18:26.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
22:18:26.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
22:18:26.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:26.065780 23403 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
22:18:26.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:26.067248 23403 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
22:18:26.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:26.067719 23403 kserver.cc:163] Server-wide thread pool size limit: 3276
22:18:26.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:26.070303 23403 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
22:18:26.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:26.074350 23403 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
22:18:26.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:26.074528 23403 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
22:18:26.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:26.074779 23403 ts_tablet_manager.cc:610] Registered 0 tablets
22:18:26.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:26.074910 23403 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
22:18:26.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:26.210997 23403 rpc_server.cc:307] RPC server started. Bound to: 127.22.73.3:45183
22:18:26.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:26.211107 23532 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.73.3:45183 every 8 connection(s)
22:18:26.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:26.213810 23403 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-2/data/info.pb
22:18:26.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:26.217795 22820 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu as pid 23403
22:18:26.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:26.218248 22820 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster1024675977643969662/ts-2/wal/instance
22:18:26.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:26.290072 23533 heartbeater.cc:347] Connected to a master server at 127.22.73.60:38817
22:18:26.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:26.290560 23533 heartbeater.cc:464] Registering TS with master...
22:18:26.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:26.291976 23533 heartbeater.cc:511] Master 127.22.73.60:38817 requested a full tablet report, sending...
22:18:26.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:26.295150 23019 ts_manager.cc:194] Registered new tserver with Master: aecd5d2d8771486fb4265da8de3b5f34 (127.22.73.3:45183)
22:18:26.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:26.299707 23534 heartbeater.cc:347] Connected to a master server at 127.22.73.61:36531
22:18:26.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:26.300081 23534 heartbeater.cc:464] Registering TS with master...
22:18:26.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:26.300899 23534 heartbeater.cc:511] Master 127.22.73.61:36531 requested a full tablet report, sending...
22:18:26.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:26.303613 22940 ts_manager.cc:194] Registered new tserver with Master: aecd5d2d8771486fb4265da8de3b5f34 (127.22.73.3:45183)
22:18:26.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:26.306845 23535 heartbeater.cc:347] Connected to a master server at 127.22.73.62:42139
22:18:26.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:26.307083 23535 heartbeater.cc:464] Registering TS with master...
22:18:26.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:26.307706 23535 heartbeater.cc:511] Master 127.22.73.62:42139 requested a full tablet report, sending...
22:18:26.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:26.309396 22864 ts_manager.cc:194] Registered new tserver with Master: aecd5d2d8771486fb4265da8de3b5f34 (127.22.73.3:45183)
22:18:26.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:26.310734 22864 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.22.73.3:42593
22:18:26.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:26.322541 22820 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
22:18:26.332 [DEBUG - main] (MiniKuduCluster.java:195) Response:
22:18:26.332 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
22:18:26.333 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.22.73.62"
port: 42139
}
bound_http_address {
host: "127.22.73.62"
port: 33791
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.22.73.61"
port: 36531
}
bound_http_address {
host: "127.22.73.61"
port: 45835
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.22.73.60"
port: 38817
}
bound_http_address {
host: "127.22.73.60"
port: 39279
}
}
}
22:18:26.334 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
22:18:26.334 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.22.73.1"
port: 35415
}
bound_http_address {
host: "127.22.73.1"
port: 33059
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.22.73.2"
port: 40829
}
bound_http_address {
host: "127.22.73.2"
port: 35867
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.22.73.3"
port: 45183
}
bound_http_address {
host: "127.22.73.3"
port: 34983
}
}
}
22:18:26.335 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
22:18:26.336 [DEBUG - main] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster120512993375883021/krb5kdc/krb5cc
22:18:26.336 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
22:18:26.337 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskyO6VHJ/test-tmp
22:18:26.338 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
22:18:26.338 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskyO6VHJ/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
22:18:26.339 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
22:18:26.659 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
22:18:26.659 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
22:18:26.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:26.662124 23544 mini_oidc.cc:152] Starting JWKS server
22:18:26.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:26.665593 23544 webserver.cc:460] Webserver started at http://127.0.0.1:37345/ using document root <none> and password file <none>
22:18:26.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:26.666072 23544 mini_oidc.cc:189] Starting OIDC Discovery server
22:18:26.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:26.667996 23544 webserver.cc:460] Webserver started at http://127.0.0.1:37403/ using document root <none> and password file <none>
22:18:26.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
22:18:26.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/krb5kdc/principal' for realm 'KRBTEST.COM',
22:18:26.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
22:18:26.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:18:26 dist-test-slave-qzth krb5kdc[23555](info): setting up network...
22:18:26.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
22:18:26.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:18:26 dist-test-slave-qzth krb5kdc[23555](info): set up 2 sockets
22:18:26.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:18:26 dist-test-slave-qzth krb5kdc[23555](info): commencing operation
22:18:26.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
22:18:27.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:27.313870 23535 heartbeater.cc:503] Master 127.22.73.62:42139 was elected leader, sending a full tablet report...
22:18:27.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:27.766227 23544 mini_kdc.cc:121] Time spent starting KDC: real 1.073s user 0.010s sys 0.007s
22:18:27.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
22:18:27.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
22:18:27.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
22:18:27.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
22:18:27.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
22:18:27.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
22:18:27.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
22:18:27.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
22:18:27.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
22:18:27.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
22:18:27.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/krb5kdc/test-user.keytab.
22:18:27.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/krb5kdc/test-user.keytab.
22:18:27.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:18:27 dist-test-slave-qzth krb5kdc[23555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547507, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
22:18:27.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
22:18:27.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T22:18:27Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
22:18:27.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T22:18:27Z Disabled control of system clock
22:18:27.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.22.254.62@KRBTEST.COM; defaulting to no policy
22:18:27.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
22:18:27.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.22.254.62@KRBTEST.COM" created.
22:18:27.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
22:18:27.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.254.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/krb5kdc/kudu_127.22.254.62.keytab.
22:18:27.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.254.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/krb5kdc/kudu_127.22.254.62.keytab.
22:18:27.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:27.940471 23544 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:18:27.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:18:27.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-0/wal
22:18:27.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-0/data
22:18:27.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
22:18:27.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
22:18:27.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
22:18:27.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:18:27.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:18:27.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:18:27.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-0/logs
22:18:27.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-0/data/info.pb
22:18:27.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:18:27.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
22:18:27.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
22:18:27.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
22:18:27.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
22:18:27.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:18:27.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:18:27.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:18:27.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
22:18:27.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
22:18:27.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:18:27.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:18:27.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.254.62:37215
22:18:27.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.254.62
22:18:27.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:18:27.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/krb5kdc/kudu_127.22.254.62.keytab
22:18:27.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.254.62
22:18:27.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
22:18:27.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
22:18:27.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
22:18:27.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.254.20:45025
22:18:27.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:18:27.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:18:27.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:18:27.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:18:27.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.254.62:37215,127.22.254.61:36153,127.22.254.60:44023
22:18:27.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:18:27.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37345/jwks.json
22:18:27.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
22:18:28.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:28.237903 23569 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
22:18:28.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:28.238490 23569 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
22:18:28.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:28.239414 23569 flags.cc:425] Enabled unsafe flag: --never_fsync=true
22:18:28.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:28.256160 23529 debug-util.cc:398] Leaking SignalData structure 0x7b08000af060 after lost signal to thread 23404
22:18:28.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:28.257000 23529 debug-util.cc:398] Leaking SignalData structure 0x7b08000aff40 after lost signal to thread 23532
22:18:28.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:28.273554 23569 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
22:18:28.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:28.273808 23569 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
22:18:28.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:28.274246 23569 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
22:18:28.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:28.274549 23569 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
22:18:28.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:28.274811 23569 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
22:18:28.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:28.275089 23569 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
22:18:28.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:28.275279 23569 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37345/jwks.json
22:18:28.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:28.321594 23569 master_runner.cc:386] Master server non-default flags:
22:18:28.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:18:28.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.254.20:45025
22:18:28.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:18:28.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:18:28.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-0/data
22:18:28.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-0/wal
22:18:28.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
22:18:28.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:18:28.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.254.62:37215,127.22.254.61:36153,127.22.254.60:44023
22:18:28.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/krb5kdc/kudu_127.22.254.62.keytab
22:18:28.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.254.62
22:18:28.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:18:28.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:18:28.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:18:28.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.254.62:37215
22:18:28.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:18:28.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
22:18:28.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:18:28.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37345/jwks.json
22:18:28.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
22:18:28.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
22:18:28.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
22:18:28.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:18:28.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:18:28.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-0/data/info.pb
22:18:28.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.254.62
22:18:28.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:18:28.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
22:18:28.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:18:28.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
22:18:28.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
22:18:28.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:18:28.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-0/logs
22:18:28.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:18:28.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
22:18:28.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:18:28.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
22:18:28.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
22:18:28.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
22:18:28.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
22:18:28.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 21:22:24 UTC on ccycloud.quasar-wysbne.root.comops.site
22:18:28.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676318
22:18:28.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
22:18:28.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:28.322682 23569 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
22:18:28.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:28.323559 23569 file_cache.cc:493] Constructed file cache file cache with capacity 419430
22:18:28.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:28.341854 23575 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:18:29.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:28.342283 23578 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:18:29.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:28.342113 23576 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:18:29.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:29.571712 23577 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
22:18:29.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:29.572301 23569 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
22:18:29.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:29.575712 23569 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
22:18:29.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:29.578526 23569 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
22:18:29.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:29.579916 23569 hybrid_clock.cc:648] HybridClock initialized: now 1790547509579851 us; error 67 us; skew 500 ppm
22:18:29.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:29.580686 23569 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
22:18:29.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:18:29 dist-test-slave-qzth krb5kdc[23555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547509, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.254.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
22:18:29.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:29.590808 23569 init.cc:377] Logged in from keytab as kudu/127.22.254.62@KRBTEST.COM (short username kudu)
22:18:29.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:29.597136 23569 webserver.cc:460] Webserver started at http://127.22.254.62:37639/ using document root <none> and password file <none>
22:18:29.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:29.598309 23569 fs_manager.cc:359] Metadata directory not provided
22:18:29.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:29.598616 23569 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:18:29.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:29.599184 23569 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
22:18:29.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:29.604063 23569 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-0/data/instance:
22:18:29.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "343dd293809c42459b96126818d74182"
22:18:29.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:18:29 on dist-test-slave-qzth"
22:18:29.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:29.605074 23569 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-0/wal/instance:
22:18:29.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "343dd293809c42459b96126818d74182"
22:18:29.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:18:29 on dist-test-slave-qzth"
22:18:29.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:29.612775 23569 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
22:18:29.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:29.618855 23586 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
22:18:29.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:29.619881 23569 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
22:18:29.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:29.620138 23569 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-0/data,/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-0/wal
22:18:29.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "343dd293809c42459b96126818d74182"
22:18:29.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:18:29 on dist-test-slave-qzth"
22:18:29.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:29.620436 23569 fs_report.cc:383] FS layout report
22:18:29.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
22:18:29.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-0/wal
22:18:29.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-0/wal
22:18:29.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-0/data/data
22:18:29.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
22:18:29.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
22:18:29.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
22:18:29.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
22:18:29.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
22:18:29.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
22:18:29.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
22:18:29.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
22:18:29.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
22:18:29.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
22:18:29.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
22:18:29.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:29.674798 23569 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
22:18:29.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:29.680467 23569 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
22:18:29.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:29.680877 23569 kserver.cc:163] Server-wide thread pool size limit: 3276
22:18:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:29.782330 23569 rpc_server.cc:307] RPC server started. Bound to: 127.22.254.62:37215
22:18:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:29.782402 23638 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.254.62:37215 every 8 connection(s)
22:18:29.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:29.785442 23569 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-0/data/info.pb
22:18:29.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:29.786988 23544 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu as pid 23569
22:18:29.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:29.787563 23544 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-0/wal/instance
22:18:29.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:29.792255 23639 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
22:18:29.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.22.254.61@KRBTEST.COM; defaulting to no policy
22:18:29.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
22:18:29.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.22.254.61@KRBTEST.COM" created.
22:18:29.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:29.809959 23639 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.254.62" port: 37215 } has no permanent_uuid. Determining permanent_uuid...
22:18:29.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
22:18:29.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.254.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/krb5kdc/kudu_127.22.254.61.keytab.
22:18:29.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.254.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/krb5kdc/kudu_127.22.254.61.keytab.
22:18:29.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:29.838558 23544 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:18:29.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:18:29.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-1/wal
22:18:29.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-1/data
22:18:29.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
22:18:29.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
22:18:29.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
22:18:29.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:18:29.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:18:29.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:18:29.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-1/logs
22:18:29.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-1/data/info.pb
22:18:29.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:18:29.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
22:18:29.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
22:18:29.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
22:18:29.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
22:18:29.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:18:29.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:18:29.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:18:29.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
22:18:29.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
22:18:29.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:18:29.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:18:29.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.254.61:36153
22:18:29.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.254.61
22:18:29.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:18:29.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/krb5kdc/kudu_127.22.254.61.keytab
22:18:29.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.254.61
22:18:29.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
22:18:29.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
22:18:29.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
22:18:29.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.254.20:45025
22:18:29.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:18:29.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:18:29.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:18:29.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:18:29.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.254.62:37215,127.22.254.61:36153,127.22.254.60:44023
22:18:29.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:18:29.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37345/jwks.json
22:18:29.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
22:18:29.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:18:29 dist-test-slave-qzth krb5kdc[23555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547509, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.254.62@KRBTEST.COM for kudu/127.22.254.62@KRBTEST.COM
22:18:29.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:29.856844 23639 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } has no permanent_uuid. Determining permanent_uuid...
22:18:29.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:29.859370 23587 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.254.61:36153: connect: Connection refused (error 111)
22:18:29.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:29.863581 23639 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.254.61:36153: Network error: Client connection negotiation failed: client connection to 127.22.254.61:36153: connect: Connection refused (error 111)
22:18:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:29.913044 23639 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } attempt: 1
22:18:29.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:29.916594 23639 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.254.61:36153: Network error: Client connection negotiation failed: client connection to 127.22.254.61:36153: connect: Connection refused (error 111)
22:18:29.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:29.985014 23639 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } attempt: 2
22:18:29.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:29.989243 23639 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.254.61:36153: Network error: Client connection negotiation failed: client connection to 127.22.254.61:36153: connect: Connection refused (error 111)
22:18:30.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:30.080694 23639 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } attempt: 3
22:18:30.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:30.084432 23639 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.254.61:36153: Network error: Client connection negotiation failed: client connection to 127.22.254.61:36153: connect: Connection refused (error 111)
22:18:30.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:30.146749 23645 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
22:18:30.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:30.147264 23645 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
22:18:30.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:30.147810 23645 flags.cc:425] Enabled unsafe flag: --never_fsync=true
22:18:30.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:30.177924 23645 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
22:18:30.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:30.178124 23645 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
22:18:30.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:30.178365 23645 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
22:18:30.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:30.178548 23645 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
22:18:30.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:30.178858 23645 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
22:18:30.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:30.179051 23645 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
22:18:30.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:30.179201 23645 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37345/jwks.json
22:18:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:30.215775 23645 master_runner.cc:386] Master server non-default flags:
22:18:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:18:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.254.20:45025
22:18:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:18:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:18:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-1/data
22:18:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-1/wal
22:18:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
22:18:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:18:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.254.62:37215,127.22.254.61:36153,127.22.254.60:44023
22:18:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/krb5kdc/kudu_127.22.254.61.keytab
22:18:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.254.61
22:18:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:18:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:18:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:18:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.254.61:36153
22:18:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:18:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
22:18:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:18:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37345/jwks.json
22:18:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
22:18:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
22:18:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
22:18:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:18:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:18:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-1/data/info.pb
22:18:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.254.61
22:18:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:18:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
22:18:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:18:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
22:18:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
22:18:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:18:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-1/logs
22:18:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:18:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
22:18:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:18:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
22:18:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
22:18:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
22:18:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
22:18:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 21:22:24 UTC on ccycloud.quasar-wysbne.root.comops.site
22:18:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676318
22:18:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
22:18:30.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:30.216821 23645 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
22:18:30.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:30.217737 23645 file_cache.cc:493] Constructed file cache file cache with capacity 419430
22:18:30.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:30.227873 23639 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } attempt: 4
22:18:30.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:30.232280 23639 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.254.61:36153: Network error: Client connection negotiation failed: client connection to 127.22.254.61:36153: connect: Connection refused (error 111)
22:18:30.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:30.236265 23655 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:18:30.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:30.531638 23639 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } attempt: 5
22:18:30.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:30.535688 23639 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.254.61:36153: Network error: Client connection negotiation failed: client connection to 127.22.254.61:36153: connect: Connection refused (error 111)
22:18:31.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:31.083053 23639 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } attempt: 6
22:18:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:31.089502 23639 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.254.61:36153: Network error: Client connection negotiation failed: client connection to 127.22.254.61:36153: connect: Connection refused (error 111)
22:18:31.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:30.236296 23653 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:18:31.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:30.237138 23652 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:18:31.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:31.446149 23654 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
22:18:31.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:31.446381 23645 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
22:18:31.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:31.449697 23645 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
22:18:31.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:31.453176 23645 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
22:18:31.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:31.454586 23645 hybrid_clock.cc:648] HybridClock initialized: now 1790547511454517 us; error 76 us; skew 500 ppm
22:18:31.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:31.455338 23645 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
22:18:31.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:18:31 dist-test-slave-qzth krb5kdc[23555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547511, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.254.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
22:18:31.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:31.461062 23645 init.cc:377] Logged in from keytab as kudu/127.22.254.61@KRBTEST.COM (short username kudu)
22:18:31.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:31.466142 23645 webserver.cc:460] Webserver started at http://127.22.254.61:33883/ using document root <none> and password file <none>
22:18:31.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:31.467006 23645 fs_manager.cc:359] Metadata directory not provided
22:18:31.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:31.467221 23645 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:18:31.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:31.467677 23645 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
22:18:31.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:31.474067 23645 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-1/data/instance:
22:18:31.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "61f1a35769dd4952861645014e55e229"
22:18:31.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:18:31 on dist-test-slave-qzth"
22:18:31.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:31.475445 23645 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-1/wal/instance:
22:18:31.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "61f1a35769dd4952861645014e55e229"
22:18:31.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:18:31 on dist-test-slave-qzth"
22:18:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:31.484211 23645 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.005s sys 0.004s
22:18:31.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:31.489717 23665 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
22:18:31.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:31.490896 23645 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s
22:18:31.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:31.491183 23645 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-1/data,/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-1/wal
22:18:31.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "61f1a35769dd4952861645014e55e229"
22:18:31.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:18:31 on dist-test-slave-qzth"
22:18:31.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:31.491464 23645 fs_report.cc:383] FS layout report
22:18:31.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
22:18:31.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-1/wal
22:18:31.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-1/wal
22:18:31.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-1/data/data
22:18:31.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
22:18:31.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
22:18:31.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
22:18:31.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
22:18:31.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
22:18:31.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
22:18:31.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
22:18:31.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
22:18:31.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
22:18:31.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
22:18:31.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
22:18:31.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:31.527464 23645 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
22:18:31.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:31.532403 23645 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
22:18:31.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:31.532845 23645 kserver.cc:163] Server-wide thread pool size limit: 3276
22:18:31.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:31.603354 23645 rpc_server.cc:307] RPC server started. Bound to: 127.22.254.61:36153
22:18:31.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:31.603468 23716 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.254.61:36153 every 8 connection(s)
22:18:31.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:31.606770 23645 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-1/data/info.pb
22:18:31.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:31.612296 23717 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
22:18:31.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:31.615593 23544 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu as pid 23645
22:18:31.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:31.615986 23544 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-1/wal/instance
22:18:31.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:31.623168 23717 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.254.62" port: 37215 } has no permanent_uuid. Determining permanent_uuid...
22:18:31.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.22.254.60@KRBTEST.COM; defaulting to no policy
22:18:31.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
22:18:31.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.22.254.60@KRBTEST.COM" created.
22:18:31.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:18:31 dist-test-slave-qzth krb5kdc[23555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547511, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.254.61@KRBTEST.COM for kudu/127.22.254.62@KRBTEST.COM
22:18:31.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
22:18:31.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.254.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/krb5kdc/kudu_127.22.254.60.keytab.
22:18:31.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.254.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/krb5kdc/kudu_127.22.254.60.keytab.
22:18:31.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:31.657970 23544 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:18:31.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:18:31.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-2/wal
22:18:31.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-2/data
22:18:31.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
22:18:31.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
22:18:31.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
22:18:31.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:18:31.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:18:31.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:18:31.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-2/logs
22:18:31.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-2/data/info.pb
22:18:31.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:18:31.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
22:18:31.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
22:18:31.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
22:18:31.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
22:18:31.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:18:31.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:18:31.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:18:31.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
22:18:31.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
22:18:31.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:18:31.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:18:31.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.254.60:44023
22:18:31.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.254.60
22:18:31.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:18:31.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/krb5kdc/kudu_127.22.254.60.keytab
22:18:31.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.254.60
22:18:31.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
22:18:31.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
22:18:31.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
22:18:31.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.254.20:45025
22:18:31.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:18:31.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:18:31.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:18:31.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:18:31.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.254.62:37215,127.22.254.61:36153,127.22.254.60:44023
22:18:31.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:18:31.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37345/jwks.json
22:18:31.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
22:18:31.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:31.659869 23717 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } has no permanent_uuid. Determining permanent_uuid...
22:18:31.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:18:31 dist-test-slave-qzth krb5kdc[23555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547511, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.254.61@KRBTEST.COM for kudu/127.22.254.61@KRBTEST.COM
22:18:31.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:31.693603 23717 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } has no permanent_uuid. Determining permanent_uuid...
22:18:31.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:31.695557 23669 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.254.60:44023: connect: Connection refused (error 111)
22:18:31.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:31.699064 23717 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.254.60:44023: Network error: Client connection negotiation failed: client connection to 127.22.254.60:44023: connect: Connection refused (error 111)
22:18:31.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:31.748481 23717 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } attempt: 1
22:18:31.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:31.752059 23717 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.254.60:44023: Network error: Client connection negotiation failed: client connection to 127.22.254.60:44023: connect: Connection refused (error 111)
22:18:31.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:31.820422 23717 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } attempt: 2
22:18:31.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:31.823937 23717 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.254.60:44023: Network error: Client connection negotiation failed: client connection to 127.22.254.60:44023: connect: Connection refused (error 111)
22:18:31.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:31.915271 23717 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } attempt: 3
22:18:31.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:31.918682 23717 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.254.60:44023: Network error: Client connection negotiation failed: client connection to 127.22.254.60:44023: connect: Connection refused (error 111)
22:18:31.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:31.971961 23723 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
22:18:31.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:31.972455 23723 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
22:18:31.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:31.972985 23723 flags.cc:425] Enabled unsafe flag: --never_fsync=true
22:18:32.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:32.018747 23723 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
22:18:32.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:32.018953 23723 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
22:18:32.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:32.019196 23723 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
22:18:32.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:32.019378 23723 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
22:18:32.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:32.019565 23723 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
22:18:32.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:32.019761 23723 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
22:18:32.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:32.019901 23723 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37345/jwks.json
22:18:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:32.056532 23723 master_runner.cc:386] Master server non-default flags:
22:18:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:18:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.254.20:45025
22:18:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:18:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:18:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-2/data
22:18:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-2/wal
22:18:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
22:18:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:18:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.254.62:37215,127.22.254.61:36153,127.22.254.60:44023
22:18:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/krb5kdc/kudu_127.22.254.60.keytab
22:18:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.254.60
22:18:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:18:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:18:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:18:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.254.60:44023
22:18:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:18:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
22:18:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:18:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37345/jwks.json
22:18:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
22:18:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
22:18:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
22:18:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:18:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:18:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-2/data/info.pb
22:18:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.254.60
22:18:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:18:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
22:18:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:18:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
22:18:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
22:18:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:18:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-2/logs
22:18:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:18:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
22:18:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:18:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
22:18:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
22:18:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
22:18:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
22:18:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 21:22:24 UTC on ccycloud.quasar-wysbne.root.comops.site
22:18:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676318
22:18:32.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
22:18:32.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:32.057600 23723 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
22:18:32.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:32.058561 23723 file_cache.cc:493] Constructed file cache file cache with capacity 419430
22:18:32.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:32.062009 23717 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } attempt: 4
22:18:32.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:32.066354 23717 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.254.60:44023: Network error: Client connection negotiation failed: client connection to 127.22.254.60:44023: connect: Connection refused (error 111)
22:18:32.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:32.076126 23732 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:18:32.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:32.150045 23639 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } attempt: 7
22:18:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:18:32 dist-test-slave-qzth krb5kdc[23555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547509, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.254.62@KRBTEST.COM for kudu/127.22.254.61@KRBTEST.COM
22:18:32.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:32.179330 23639 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } has no permanent_uuid. Determining permanent_uuid...
22:18:32.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:32.184895 23639 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.254.60:44023: Network error: Client connection negotiation failed: client connection to 127.22.254.60:44023: connect: Connection refused (error 111)
22:18:32.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:32.243276 23639 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } attempt: 1
22:18:32.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:32.247215 23639 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.254.60:44023: Network error: Client connection negotiation failed: client connection to 127.22.254.60:44023: connect: Connection refused (error 111)
22:18:32.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:32.328780 23639 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } attempt: 2
22:18:32.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:32.332631 23639 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.254.60:44023: Network error: Client connection negotiation failed: client connection to 127.22.254.60:44023: connect: Connection refused (error 111)
22:18:32.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:32.365864 23717 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } attempt: 5
22:18:32.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:32.369450 23717 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.254.60:44023: Network error: Client connection negotiation failed: client connection to 127.22.254.60:44023: connect: Connection refused (error 111)
22:18:32.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:32.418151 23639 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } attempt: 3
22:18:32.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:32.421972 23639 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.254.60:44023: Network error: Client connection negotiation failed: client connection to 127.22.254.60:44023: connect: Connection refused (error 111)
22:18:32.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:32.562529 23639 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } attempt: 4
22:18:32.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:32.568198 23639 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.254.60:44023: Network error: Client connection negotiation failed: client connection to 127.22.254.60:44023: connect: Connection refused (error 111)
22:18:32.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:32.851603 23639 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } attempt: 5
22:18:32.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:32.856029 23639 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.254.60:44023: Network error: Client connection negotiation failed: client connection to 127.22.254.60:44023: connect: Connection refused (error 111)
22:18:32.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:32.916790 23717 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } attempt: 6
22:18:32.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:32.923251 23717 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.254.60:44023: Network error: Client connection negotiation failed: client connection to 127.22.254.60:44023: connect: Connection refused (error 111)
22:18:33.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:32.076161 23731 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:18:33.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:32.077467 23734 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:18:33.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:32.077347 23723 server_base.cc:1034] running on GCE node
22:18:33.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.293891 23723 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
22:18:33.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.296324 23723 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
22:18:33.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.297643 23723 hybrid_clock.cc:648] HybridClock initialized: now 1790547513297610 us; error 43 us; skew 500 ppm
22:18:33.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.298446 23723 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
22:18:33.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:18:33 dist-test-slave-qzth krb5kdc[23555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547513, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.254.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
22:18:33.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.304052 23723 init.cc:377] Logged in from keytab as kudu/127.22.254.60@KRBTEST.COM (short username kudu)
22:18:33.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.308475 23723 webserver.cc:460] Webserver started at http://127.22.254.60:41665/ using document root <none> and password file <none>
22:18:33.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.309490 23723 fs_manager.cc:359] Metadata directory not provided
22:18:33.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.309726 23723 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:18:33.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.310150 23723 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
22:18:33.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.314771 23723 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-2/data/instance:
22:18:33.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0cb22a9de93b4bfb93a644663cb469af"
22:18:33.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:18:33 on dist-test-slave-qzth"
22:18:33.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.315883 23723 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-2/wal/instance:
22:18:33.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0cb22a9de93b4bfb93a644663cb469af"
22:18:33.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:18:33 on dist-test-slave-qzth"
22:18:33.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.323055 23723 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
22:18:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.328554 23747 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
22:18:33.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.329521 23723 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.001s
22:18:33.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.329764 23723 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-2/data,/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-2/wal
22:18:33.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0cb22a9de93b4bfb93a644663cb469af"
22:18:33.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:18:33 on dist-test-slave-qzth"
22:18:33.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.330175 23723 fs_report.cc:383] FS layout report
22:18:33.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
22:18:33.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-2/wal
22:18:33.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-2/wal
22:18:33.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-2/data/data
22:18:33.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
22:18:33.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
22:18:33.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
22:18:33.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
22:18:33.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
22:18:33.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
22:18:33.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
22:18:33.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
22:18:33.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
22:18:33.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
22:18:33.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
22:18:33.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.369314 23723 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
22:18:33.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.374785 23723 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
22:18:33.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.375154 23723 kserver.cc:163] Server-wide thread pool size limit: 3276
22:18:33.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.408432 23639 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } attempt: 6
22:18:33.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:33.415410 23639 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.254.60:44023: Network error: Client connection negotiation failed: client connection to 127.22.254.60:44023: connect: Connection refused (error 111)
22:18:33.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.446938 23723 rpc_server.cc:307] RPC server started. Bound to: 127.22.254.60:44023
22:18:33.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.447031 23800 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.254.60:44023 every 8 connection(s)
22:18:33.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.449294 23723 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-2/data/info.pb
22:18:33.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.454272 23544 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu as pid 23723
22:18:33.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.454735 23544 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-2/wal/instance
22:18:33.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.454903 23801 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
22:18:33.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.467517 23801 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.254.62" port: 37215 } has no permanent_uuid. Determining permanent_uuid...
22:18:33.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:18:33 dist-test-slave-qzth krb5kdc[23555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547507, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.22.254.60@KRBTEST.COM
22:18:33.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:18:33 dist-test-slave-qzth krb5kdc[23555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547513, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.254.60@KRBTEST.COM for kudu/127.22.254.62@KRBTEST.COM
22:18:33.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.507102 23801 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } has no permanent_uuid. Determining permanent_uuid...
22:18:33.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:18:33 dist-test-slave-qzth krb5kdc[23555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547513, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.254.60@KRBTEST.COM for kudu/127.22.254.61@KRBTEST.COM
22:18:33.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:18:33 dist-test-slave-qzth krb5kdc[23555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547507, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.22.254.61@KRBTEST.COM
22:18:33.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.532514 23801 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } has no permanent_uuid. Determining permanent_uuid...
22:18:33.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:18:33 dist-test-slave-qzth krb5kdc[23555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547513, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.254.60@KRBTEST.COM for kudu/127.22.254.60@KRBTEST.COM
22:18:33.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:18:33 dist-test-slave-qzth krb5kdc[23555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547507, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.22.254.62@KRBTEST.COM
22:18:33.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.564738 23801 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af: Bootstrap starting.
22:18:33.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.571501 23801 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af: Neither blocks nor log segments found. Creating new log.
22:18:33.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.573130 23544 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
22:18:33.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.573437 23801 log.cc:824] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af: Log is configured to *not* fsync() on all Append() calls
22:18:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.578053 23801 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af: No bootstrap required, opened a new log
22:18:33.581 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
22:18:33.581 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
22:18:33.582 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
22:18:33.582 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
22:18:33.584 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.22.254.62"
port: 37215
}
bound_http_address {
host: "127.22.254.62"
port: 37639
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.22.254.61"
port: 36153
}
bound_http_address {
host: "127.22.254.61"
port: 33883
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.22.254.60"
port: 44023
}
bound_http_address {
host: "127.22.254.60"
port: 41665
}
}
}
22:18:33.584 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
22:18:33.584 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
22:18:33.584 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
22:18:33.585 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
22:18:33.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.604455 23801 raft_consensus.cc:348] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "343dd293809c42459b96126818d74182" member_type: VOTER last_known_addr { host: "127.22.254.62" port: 37215 } } peers { permanent_uuid: "61f1a35769dd4952861645014e55e229" member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } } peers { permanent_uuid: "0cb22a9de93b4bfb93a644663cb469af" member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } }
22:18:33.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.605057 23801 raft_consensus.cc:374] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
22:18:33.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.605324 23801 raft_consensus.cc:729] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0cb22a9de93b4bfb93a644663cb469af, State: Initialized, Role: FOLLOWER
22:18:33.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.606256 23801 consensus_queue.cc:260] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "343dd293809c42459b96126818d74182" member_type: VOTER last_known_addr { host: "127.22.254.62" port: 37215 } } peers { permanent_uuid: "61f1a35769dd4952861645014e55e229" member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } } peers { permanent_uuid: "0cb22a9de93b4bfb93a644663cb469af" member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } }
22:18:33.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.609347 23813 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "343dd293809c42459b96126818d74182" member_type: VOTER last_known_addr { host: "127.22.254.62" port: 37215 } } peers { permanent_uuid: "61f1a35769dd4952861645014e55e229" member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } } peers { permanent_uuid: "0cb22a9de93b4bfb93a644663cb469af" member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } } }
22:18:33.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.610849 23813 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af [sys.catalog]: This master's current role is: FOLLOWER
22:18:33.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.612726 23801 sys_catalog.cc:564] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af [sys.catalog]: configured and running, proceeding with master startup.
22:18:33.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:18:33 dist-test-slave-qzth krb5kdc[23555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547513, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
22:18:33.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
22:18:33.631 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
22:18:33.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:33.630360 23753 tablet.cc:2369] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af: Can't schedule compaction. Clean time has not been advanced past its initial value.
22:18:33.632 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/krb5kdc/krb5cc
22:18:33.633 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
22:18:33.633 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/krb5kdc/krb5cc
22:18:33.634 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
22:18:33.635 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
22:18:33.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:33.646926 23824 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:18:33.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:33.647181 23824 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:18:33.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.652659 23813 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
22:18:33.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.653172 23813 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "343dd293809c42459b96126818d74182" member_type: VOTER last_known_addr { host: "127.22.254.62" port: 37215 } } peers { permanent_uuid: "61f1a35769dd4952861645014e55e229" member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } } peers { permanent_uuid: "0cb22a9de93b4bfb93a644663cb469af" member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } }
22:18:33.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.655361 23813 leader_election.cc:288] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 343dd293809c42459b96126818d74182 (127.22.254.62:37215), 61f1a35769dd4952861645014e55e229 (127.22.254.61:36153)
22:18:33.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.644845 23723 master_runner.cc:186] Error getting master registration for 127.22.254.62:37215: OK, instance_id { permanent_uuid: "343dd293809c42459b96126818d74182" instance_seqno: 1790547509758746 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
22:18:33.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.677429 23723 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
22:18:33.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:33.983623 23717 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } attempt: 7
22:18:33.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:18:33 dist-test-slave-qzth krb5kdc[23555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547511, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.254.61@KRBTEST.COM for kudu/127.22.254.60@KRBTEST.COM
22:18:34.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.022115 23717 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 61f1a35769dd4952861645014e55e229: Bootstrap starting.
22:18:34.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.029413 23717 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 61f1a35769dd4952861645014e55e229: Neither blocks nor log segments found. Creating new log.
22:18:34.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.031952 23717 log.cc:824] T 00000000000000000000000000000000 P 61f1a35769dd4952861645014e55e229: Log is configured to *not* fsync() on all Append() calls
22:18:34.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.037425 23717 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 61f1a35769dd4952861645014e55e229: No bootstrap required, opened a new log
22:18:34.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.068532 23717 raft_consensus.cc:348] T 00000000000000000000000000000000 P 61f1a35769dd4952861645014e55e229 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "343dd293809c42459b96126818d74182" member_type: VOTER last_known_addr { host: "127.22.254.62" port: 37215 } } peers { permanent_uuid: "61f1a35769dd4952861645014e55e229" member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } } peers { permanent_uuid: "0cb22a9de93b4bfb93a644663cb469af" member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } }
22:18:34.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.069347 23717 raft_consensus.cc:374] T 00000000000000000000000000000000 P 61f1a35769dd4952861645014e55e229 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
22:18:34.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.069631 23717 raft_consensus.cc:729] T 00000000000000000000000000000000 P 61f1a35769dd4952861645014e55e229 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 61f1a35769dd4952861645014e55e229, State: Initialized, Role: FOLLOWER
22:18:34.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.070554 23717 consensus_queue.cc:260] T 00000000000000000000000000000000 P 61f1a35769dd4952861645014e55e229 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "343dd293809c42459b96126818d74182" member_type: VOTER last_known_addr { host: "127.22.254.62" port: 37215 } } peers { permanent_uuid: "61f1a35769dd4952861645014e55e229" member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } } peers { permanent_uuid: "0cb22a9de93b4bfb93a644663cb469af" member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } }
22:18:34.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.074090 23829 sys_catalog.cc:455] T 00000000000000000000000000000000 P 61f1a35769dd4952861645014e55e229 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "343dd293809c42459b96126818d74182" member_type: VOTER last_known_addr { host: "127.22.254.62" port: 37215 } } peers { permanent_uuid: "61f1a35769dd4952861645014e55e229" member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } } peers { permanent_uuid: "0cb22a9de93b4bfb93a644663cb469af" member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } } }
22:18:34.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.075522 23829 sys_catalog.cc:458] T 00000000000000000000000000000000 P 61f1a35769dd4952861645014e55e229 [sys.catalog]: This master's current role is: FOLLOWER
22:18:34.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.075532 23717 sys_catalog.cc:564] T 00000000000000000000000000000000 P 61f1a35769dd4952861645014e55e229 [sys.catalog]: configured and running, proceeding with master startup.
22:18:34.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.075739 23692 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0cb22a9de93b4bfb93a644663cb469af" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "61f1a35769dd4952861645014e55e229" is_pre_election: true
22:18:34.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.076823 23692 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 61f1a35769dd4952861645014e55e229 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0cb22a9de93b4bfb93a644663cb469af in term 0.
22:18:34.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.078219 23748 leader_election.cc:302] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0cb22a9de93b4bfb93a644663cb469af, 61f1a35769dd4952861645014e55e229; no voters:
22:18:34.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.079159 23813 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af [term 0 FOLLOWER]: Leader pre-election won for term 1
22:18:34.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.079564 23813 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
22:18:34.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.079905 23813 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af [term 0 FOLLOWER]: Advancing to term 1
22:18:34.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.091496 23813 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "343dd293809c42459b96126818d74182" member_type: VOTER last_known_addr { host: "127.22.254.62" port: 37215 } } peers { permanent_uuid: "61f1a35769dd4952861645014e55e229" member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } } peers { permanent_uuid: "0cb22a9de93b4bfb93a644663cb469af" member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } }
22:18:34.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.097358 23692 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0cb22a9de93b4bfb93a644663cb469af" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "61f1a35769dd4952861645014e55e229"
22:18:34.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.097942 23692 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 61f1a35769dd4952861645014e55e229 [term 0 FOLLOWER]: Advancing to term 1
22:18:34.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.098609 23813 leader_election.cc:288] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af [CANDIDATE]: Term 1 election: Requested vote from peers 343dd293809c42459b96126818d74182 (127.22.254.62:37215), 61f1a35769dd4952861645014e55e229 (127.22.254.61:36153)
22:18:34.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:34.110180 23840 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 61f1a35769dd4952861645014e55e229: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:18:34.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:34.110711 23840 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:18:34.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.112788 23692 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 61f1a35769dd4952861645014e55e229 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0cb22a9de93b4bfb93a644663cb469af in term 1.
22:18:34.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.113725 23748 leader_election.cc:302] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0cb22a9de93b4bfb93a644663cb469af, 61f1a35769dd4952861645014e55e229; no voters:
22:18:34.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.114681 23813 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af [term 1 FOLLOWER]: Leader election won for term 1
22:18:34.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.119894 23813 raft_consensus.cc:686] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af [term 1 LEADER]: Becoming Leader. State: Replica: 0cb22a9de93b4bfb93a644663cb469af, State: Running, Role: LEADER
22:18:34.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.122099 23813 consensus_queue.cc:237] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "343dd293809c42459b96126818d74182" member_type: VOTER last_known_addr { host: "127.22.254.62" port: 37215 } } peers { permanent_uuid: "61f1a35769dd4952861645014e55e229" member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } } peers { permanent_uuid: "0cb22a9de93b4bfb93a644663cb469af" member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } }
22:18:34.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.103045 23645 master_runner.cc:186] Error getting master registration for 127.22.254.62:37215: OK, instance_id { permanent_uuid: "343dd293809c42459b96126818d74182" instance_seqno: 1790547509758746 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
22:18:34.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.138303 23645 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
22:18:34.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.145382 23842 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0cb22a9de93b4bfb93a644663cb469af. Latest consensus state: current_term: 1 leader_uuid: "0cb22a9de93b4bfb93a644663cb469af" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "343dd293809c42459b96126818d74182" member_type: VOTER last_known_addr { host: "127.22.254.62" port: 37215 } } peers { permanent_uuid: "61f1a35769dd4952861645014e55e229" member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } } peers { permanent_uuid: "0cb22a9de93b4bfb93a644663cb469af" member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } } }
22:18:34.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.146142 23842 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af [sys.catalog]: This master's current role is: LEADER
22:18:34.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.155251 23844 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
22:18:34.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.162348 23844 catalog_manager.cc:1484] Initializing Kudu cluster ID...
22:18:34.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.179467 23692 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 61f1a35769dd4952861645014e55e229 [term 1 FOLLOWER]: Refusing update from remote peer 0cb22a9de93b4bfb93a644663cb469af: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
22:18:34.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.181563 23842 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af [LEADER]: Connected to new peer: Peer: permanent_uuid: "61f1a35769dd4952861645014e55e229" member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
22:18:34.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.211478 23845 mvcc.cc:205] Tried to move back new op lower bound from 7334082618062700544 to 7334082617867911168. Current Snapshot: MvccSnapshot[applied={T|T < 7334082618062700544}]
22:18:34.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.214697 23844 catalog_manager.cc:1347] Generated new cluster ID: 3feed9418f774d00b5c787fb3c6c3065
22:18:34.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.214949 23844 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
22:18:34.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.219012 23813 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "0cb22a9de93b4bfb93a644663cb469af" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "343dd293809c42459b96126818d74182" member_type: VOTER last_known_addr { host: "127.22.254.62" port: 37215 } } peers { permanent_uuid: "61f1a35769dd4952861645014e55e229" member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } } peers { permanent_uuid: "0cb22a9de93b4bfb93a644663cb469af" member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } } }
22:18:34.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.220845 23813 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af [sys.catalog]: This master's current role is: LEADER
22:18:34.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.224710 23829 sys_catalog.cc:455] T 00000000000000000000000000000000 P 61f1a35769dd4952861645014e55e229 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0cb22a9de93b4bfb93a644663cb469af. Latest consensus state: current_term: 1 leader_uuid: "0cb22a9de93b4bfb93a644663cb469af" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "343dd293809c42459b96126818d74182" member_type: VOTER last_known_addr { host: "127.22.254.62" port: 37215 } } peers { permanent_uuid: "61f1a35769dd4952861645014e55e229" member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } } peers { permanent_uuid: "0cb22a9de93b4bfb93a644663cb469af" member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } } }
22:18:34.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.226574 23829 sys_catalog.cc:458] T 00000000000000000000000000000000 P 61f1a35769dd4952861645014e55e229 [sys.catalog]: This master's current role is: FOLLOWER
22:18:34.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.231979 23829 sys_catalog.cc:455] T 00000000000000000000000000000000 P 61f1a35769dd4952861645014e55e229 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "0cb22a9de93b4bfb93a644663cb469af" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "343dd293809c42459b96126818d74182" member_type: VOTER last_known_addr { host: "127.22.254.62" port: 37215 } } peers { permanent_uuid: "61f1a35769dd4952861645014e55e229" member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } } peers { permanent_uuid: "0cb22a9de93b4bfb93a644663cb469af" member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } } }
22:18:34.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.232573 23829 sys_catalog.cc:458] T 00000000000000000000000000000000 P 61f1a35769dd4952861645014e55e229 [sys.catalog]: This master's current role is: FOLLOWER
22:18:34.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.257104 23844 catalog_manager.cc:1370] Generated new certificate authority record
22:18:34.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.261161 23844 catalog_manager.cc:1504] Loading token signing keys...
22:18:34.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.286881 23844 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af: Generated new TSK 0
22:18:34.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.287942 23844 catalog_manager.cc:1514] Initializing in-progress tserver states...
22:18:34.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.448871 23639 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } attempt: 7
22:18:34.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:18:34 dist-test-slave-qzth krb5kdc[23555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547509, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.254.62@KRBTEST.COM for kudu/127.22.254.60@KRBTEST.COM
22:18:34.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.526574 23639 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182: Bootstrap starting.
22:18:34.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.544358 23639 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182: Neither blocks nor log segments found. Creating new log.
22:18:34.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.551193 23639 log.cc:824] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182: Log is configured to *not* fsync() on all Append() calls
22:18:34.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.564160 23639 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182: No bootstrap required, opened a new log
22:18:34.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.651127 23639 raft_consensus.cc:348] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "343dd293809c42459b96126818d74182" member_type: VOTER last_known_addr { host: "127.22.254.62" port: 37215 } } peers { permanent_uuid: "61f1a35769dd4952861645014e55e229" member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } } peers { permanent_uuid: "0cb22a9de93b4bfb93a644663cb469af" member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } }
22:18:34.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.654821 23639 raft_consensus.cc:374] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
22:18:34.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.655644 23639 raft_consensus.cc:729] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 343dd293809c42459b96126818d74182, State: Initialized, Role: FOLLOWER
22:18:34.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.656885 23639 consensus_queue.cc:260] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "343dd293809c42459b96126818d74182" member_type: VOTER last_known_addr { host: "127.22.254.62" port: 37215 } } peers { permanent_uuid: "61f1a35769dd4952861645014e55e229" member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } } peers { permanent_uuid: "0cb22a9de93b4bfb93a644663cb469af" member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } }
22:18:34.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.660799 23853 sys_catalog.cc:455] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "343dd293809c42459b96126818d74182" member_type: VOTER last_known_addr { host: "127.22.254.62" port: 37215 } } peers { permanent_uuid: "61f1a35769dd4952861645014e55e229" member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } } peers { permanent_uuid: "0cb22a9de93b4bfb93a644663cb469af" member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } } }
22:18:34.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.662889 23853 sys_catalog.cc:458] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [sys.catalog]: This master's current role is: FOLLOWER
22:18:34.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.665858 23639 sys_catalog.cc:564] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [sys.catalog]: configured and running, proceeding with master startup.
22:18:34.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.668138 23614 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0cb22a9de93b4bfb93a644663cb469af" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "343dd293809c42459b96126818d74182" is_pre_election: true
22:18:34.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.668998 23614 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0cb22a9de93b4bfb93a644663cb469af in term 0.
22:18:34.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.669461 23614 rpcz_store.cc:275] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:56644 (request call id 2) took 1013 ms. Trace:
22:18:34.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.670547 23612 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [term 0 FOLLOWER]: Advancing to term 1
22:18:34.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.674031 23613 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0cb22a9de93b4bfb93a644663cb469af" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "343dd293809c42459b96126818d74182"
22:18:34.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.669746 23614 rpcz_store.cc:276] 0927 22:18:33.655999 (+ 0us) service_pool.cc:170] Inserting onto call queue
22:18:34.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:33.656139 (+ 140us) service_pool.cc:227] Handling call
22:18:34.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:34.669400 (+1013261us) inbound_call.cc:173] Queueing success response
22:18:34.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
22:18:34.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.685032 23612 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [term 1 FOLLOWER]: Refusing update from remote peer 0cb22a9de93b4bfb93a644663cb469af: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
22:18:34.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:34.691874 23592 tablet.cc:2369] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182: Can't schedule compaction. Clean time has not been advanced past its initial value.
22:18:34.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.699927 23842 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af [LEADER]: Connected to new peer: Peer: permanent_uuid: "343dd293809c42459b96126818d74182" member_type: VOTER last_known_addr { host: "127.22.254.62" port: 37215 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
22:18:34.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.748440 23853 sys_catalog.cc:455] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0cb22a9de93b4bfb93a644663cb469af. Latest consensus state: current_term: 1 leader_uuid: "0cb22a9de93b4bfb93a644663cb469af" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "343dd293809c42459b96126818d74182" member_type: VOTER last_known_addr { host: "127.22.254.62" port: 37215 } } peers { permanent_uuid: "61f1a35769dd4952861645014e55e229" member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } } peers { permanent_uuid: "0cb22a9de93b4bfb93a644663cb469af" member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } } }
22:18:34.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.749466 23853 sys_catalog.cc:458] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [sys.catalog]: This master's current role is: FOLLOWER
22:18:34.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.750830 23863 sys_catalog.cc:455] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "0cb22a9de93b4bfb93a644663cb469af" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "343dd293809c42459b96126818d74182" member_type: VOTER last_known_addr { host: "127.22.254.62" port: 37215 } } peers { permanent_uuid: "61f1a35769dd4952861645014e55e229" member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } } peers { permanent_uuid: "0cb22a9de93b4bfb93a644663cb469af" member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } } }
22:18:34.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.751690 23863 sys_catalog.cc:458] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [sys.catalog]: This master's current role is: FOLLOWER
22:18:34.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.794548 23870 catalog_manager.cc:1259] Loaded cluster ID: 3feed9418f774d00b5c787fb3c6c3065
22:18:34.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.794975 23870 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182: loading cluster ID for follower catalog manager: success
22:18:34.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.802644 23870 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182: acquiring CA information for follower catalog manager: success
22:18:34.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:34.807790 23870 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
22:18:35.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:35.116997 23840 catalog_manager.cc:1259] Loaded cluster ID: 3feed9418f774d00b5c787fb3c6c3065
22:18:35.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:35.117489 23840 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 61f1a35769dd4952861645014e55e229: loading cluster ID for follower catalog manager: success
22:18:35.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:35.125041 23840 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 61f1a35769dd4952861645014e55e229: acquiring CA information for follower catalog manager: success
22:18:35.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:35.131305 23840 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 61f1a35769dd4952861645014e55e229: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
22:18:35.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:35.162596 23842 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "0cb22a9de93b4bfb93a644663cb469af" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "343dd293809c42459b96126818d74182" member_type: VOTER last_known_addr { host: "127.22.254.62" port: 37215 } } peers { permanent_uuid: "61f1a35769dd4952861645014e55e229" member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } } peers { permanent_uuid: "0cb22a9de93b4bfb93a644663cb469af" member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } } }
22:18:35.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:35.164525 23842 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af [sys.catalog]: This master's current role is: LEADER
22:18:36.635 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.22.254.62:37215
22:18:36.635 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
22:18:36.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:36.636639 23544 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu with pid 23569
22:18:36.665 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
22:18:36.666 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.22.254.60:44023
22:18:36.666 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
22:18:36.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:36.666580 23544 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu with pid 23723
22:18:36.702 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
22:18:36.704 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.22.254.61:36153
22:18:36.704 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
22:18:36.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:36.704924 23544 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu with pid 23645
22:18:36.736 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
22:18:36.737 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.22.254.62:37215
22:18:36.737 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
22:18:36.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:36.738662 23544 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:18:36.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:18:36.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-0/wal
22:18:36.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-0/data
22:18:36.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
22:18:36.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
22:18:36.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
22:18:36.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:18:36.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:18:36.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:18:36.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-0/logs
22:18:36.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-0/data/info.pb
22:18:36.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:18:36.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
22:18:36.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
22:18:36.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
22:18:36.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
22:18:36.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:18:36.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:18:36.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:18:36.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
22:18:36.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
22:18:36.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:18:36.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:18:36.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.254.62:37215
22:18:36.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.254.62
22:18:36.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37639
22:18:36.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/krb5kdc/kudu_127.22.254.62.keytab
22:18:36.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.254.62
22:18:36.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
22:18:36.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
22:18:36.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
22:18:36.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.254.20:45025
22:18:36.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:18:36.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:18:36.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:18:36.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:18:36.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.254.62:37215,127.22.254.61:36153,127.22.254.60:44023
22:18:36.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:18:36.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37345/jwks.json
22:18:36.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
22:18:36.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:36.780169 22971 debug-util.cc:398] Leaking SignalData structure 0x7b08000aeec0 after lost signal to thread 22906
22:18:36.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:36.780938 22971 debug-util.cc:398] Leaking SignalData structure 0x7b08000a5200 after lost signal to thread 22974
22:18:37.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:37.061568 23878 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
22:18:37.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:37.062275 23878 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
22:18:37.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:37.063037 23878 flags.cc:425] Enabled unsafe flag: --never_fsync=true
22:18:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:37.095835 23878 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
22:18:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:37.096117 23878 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
22:18:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:37.096514 23878 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
22:18:37.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:37.096835 23878 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
22:18:37.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:37.097128 23878 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
22:18:37.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:37.097415 23878 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
22:18:37.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:37.097625 23878 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37345/jwks.json
22:18:37.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:37.134532 23878 master_runner.cc:386] Master server non-default flags:
22:18:37.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:18:37.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.254.20:45025
22:18:37.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:18:37.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:18:37.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-0/data
22:18:37.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-0/wal
22:18:37.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
22:18:37.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:18:37.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.254.62:37215,127.22.254.61:36153,127.22.254.60:44023
22:18:37.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/krb5kdc/kudu_127.22.254.62.keytab
22:18:37.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.254.62
22:18:37.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:18:37.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:18:37.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:18:37.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.254.62:37215
22:18:37.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:18:37.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
22:18:37.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:18:37.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37345/jwks.json
22:18:37.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
22:18:37.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
22:18:37.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
22:18:37.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:18:37.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:18:37.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-0/data/info.pb
22:18:37.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.254.62
22:18:37.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37639
22:18:37.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
22:18:37.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:18:37.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
22:18:37.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
22:18:37.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:18:37.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-0/logs
22:18:37.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:18:37.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
22:18:37.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:18:37.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
22:18:37.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
22:18:37.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
22:18:37.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
22:18:37.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 21:22:24 UTC on ccycloud.quasar-wysbne.root.comops.site
22:18:37.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676318
22:18:37.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
22:18:37.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:37.135936 23878 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
22:18:37.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:37.137127 23878 file_cache.cc:493] Constructed file cache file cache with capacity 419430
22:18:37.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:37.157084 23885 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:18:38.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:37.157202 23884 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:18:38.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:37.160259 23887 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:18:38.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:38.719403 23886 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1560 milliseconds
22:18:38.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:38.719549 23878 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
22:18:38.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:38.721115 23878 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
22:18:38.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:38.724381 23878 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
22:18:38.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:38.725837 23878 hybrid_clock.cc:648] HybridClock initialized: now 1790547518725725 us; error 101 us; skew 500 ppm
22:18:38.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:38.727046 23878 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
22:18:38.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:18:38 dist-test-slave-qzth krb5kdc[23555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547518, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.254.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
22:18:38.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:38.734586 23878 init.cc:377] Logged in from keytab as kudu/127.22.254.62@KRBTEST.COM (short username kudu)
22:18:38.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:38.740305 23878 webserver.cc:460] Webserver started at http://127.22.254.62:37639/ using document root <none> and password file <none>
22:18:38.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:38.741628 23878 fs_manager.cc:359] Metadata directory not provided
22:18:38.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:38.741971 23878 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:18:38.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:38.753168 23878 fs_manager.cc:709] Time spent opening directory manager: real 0.007s user 0.006s sys 0.000s
22:18:38.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:38.758841 23896 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
22:18:38.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:38.760108 23878 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
22:18:38.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:38.760499 23878 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-0/data,/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-0/wal
22:18:38.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "343dd293809c42459b96126818d74182"
22:18:38.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:18:29 on dist-test-slave-qzth"
22:18:38.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:38.763906 23878 fs_report.cc:383] FS layout report
22:18:38.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
22:18:38.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-0/wal
22:18:38.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-0/wal
22:18:38.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-0/data/data
22:18:38.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
22:18:38.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
22:18:38.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
22:18:38.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
22:18:38.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
22:18:38.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
22:18:38.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
22:18:38.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
22:18:38.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
22:18:38.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
22:18:38.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
22:18:38.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:38.807278 23878 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
22:18:38.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:38.812894 23878 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
22:18:38.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:38.813375 23878 kserver.cc:163] Server-wide thread pool size limit: 3276
22:18:38.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:38.900290 23878 rpc_server.cc:307] RPC server started. Bound to: 127.22.254.62:37215
22:18:38.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:38.900413 23947 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.254.62:37215 every 8 connection(s)
22:18:38.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:38.903510 23878 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-0/data/info.pb
22:18:38.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:38.908877 23544 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu as pid 23878
22:18:38.909 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
22:18:38.909 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.22.254.60:44023
22:18:38.909 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
22:18:38.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:38.911501 23544 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:18:38.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:18:38.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-2/wal
22:18:38.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-2/data
22:18:38.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
22:18:38.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
22:18:38.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
22:18:38.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:18:38.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:18:38.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:18:38.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-2/logs
22:18:38.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-2/data/info.pb
22:18:38.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:18:38.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
22:18:38.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
22:18:38.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
22:18:38.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
22:18:38.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:18:38.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:18:38.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:18:38.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
22:18:38.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
22:18:38.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:18:38.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:18:38.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.254.60:44023
22:18:38.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.254.60
22:18:38.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41665
22:18:38.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/krb5kdc/kudu_127.22.254.60.keytab
22:18:38.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.254.60
22:18:38.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
22:18:38.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
22:18:38.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
22:18:38.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.254.20:45025
22:18:38.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:18:38.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:18:38.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:18:38.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:18:38.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.254.62:37215,127.22.254.61:36153,127.22.254.60:44023
22:18:38.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:18:38.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37345/jwks.json
22:18:38.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
22:18:38.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:38.921213 23948 sys_catalog.cc:263] Verifying existing consensus state
22:18:38.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:38.926523 23948 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182: Bootstrap starting.
22:18:38.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:38.962438 23948 log.cc:824] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182: Log is configured to *not* fsync() on all Append() calls
22:18:38.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:38.977985 23948 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
22:18:38.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:38.978856 23948 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182: Bootstrap complete.
22:18:39.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:39.003587 23948 raft_consensus.cc:348] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "343dd293809c42459b96126818d74182" member_type: VOTER last_known_addr { host: "127.22.254.62" port: 37215 } } peers { permanent_uuid: "61f1a35769dd4952861645014e55e229" member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } } peers { permanent_uuid: "0cb22a9de93b4bfb93a644663cb469af" member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } }
22:18:39.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:39.005903 23948 raft_consensus.cc:729] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 343dd293809c42459b96126818d74182, State: Initialized, Role: FOLLOWER
22:18:39.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:39.006807 23948 consensus_queue.cc:260] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "343dd293809c42459b96126818d74182" member_type: VOTER last_known_addr { host: "127.22.254.62" port: 37215 } } peers { permanent_uuid: "61f1a35769dd4952861645014e55e229" member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } } peers { permanent_uuid: "0cb22a9de93b4bfb93a644663cb469af" member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } }
22:18:39.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:39.009846 23952 sys_catalog.cc:455] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "343dd293809c42459b96126818d74182" member_type: VOTER last_known_addr { host: "127.22.254.62" port: 37215 } } peers { permanent_uuid: "61f1a35769dd4952861645014e55e229" member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } } peers { permanent_uuid: "0cb22a9de93b4bfb93a644663cb469af" member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } } }
22:18:39.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:39.010780 23952 sys_catalog.cc:458] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [sys.catalog]: This master's current role is: FOLLOWER
22:18:39.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:39.011543 23948 sys_catalog.cc:564] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [sys.catalog]: configured and running, proceeding with master startup.
22:18:39.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:39.060253 23963 catalog_manager.cc:1259] Loaded cluster ID: 3feed9418f774d00b5c787fb3c6c3065
22:18:39.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:39.060709 23963 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182: loading cluster ID for follower catalog manager: success
22:18:39.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:39.068738 23963 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182: acquiring CA information for follower catalog manager: success
22:18:39.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:18:39 dist-test-slave-qzth krb5kdc[23555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547518, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.254.62@KRBTEST.COM for kudu/127.22.254.62@KRBTEST.COM
22:18:39.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:39.093357 23963 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
22:18:39.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:39.108570 23878 master_runner.cc:186] Error getting master registration for 127.22.254.61:36153: Network error: Client connection negotiation failed: client connection to 127.22.254.61:36153: connect: Connection refused (error 111),
22:18:39.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:39.136193 23878 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
22:18:39.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:39.319607 23950 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
22:18:39.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:39.320308 23950 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
22:18:39.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:39.321165 23950 flags.cc:425] Enabled unsafe flag: --never_fsync=true
22:18:39.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:39.351850 23950 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
22:18:39.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:39.352118 23950 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
22:18:39.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:39.352599 23950 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
22:18:39.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:39.352985 23950 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
22:18:39.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:39.353289 23950 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
22:18:39.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:39.353571 23950 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
22:18:39.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:39.353842 23950 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37345/jwks.json
22:18:39.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:39.401157 23950 master_runner.cc:386] Master server non-default flags:
22:18:39.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:18:39.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.254.20:45025
22:18:39.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:18:39.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:18:39.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-2/data
22:18:39.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-2/wal
22:18:39.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
22:18:39.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:18:39.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.254.62:37215,127.22.254.61:36153,127.22.254.60:44023
22:18:39.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/krb5kdc/kudu_127.22.254.60.keytab
22:18:39.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.254.60
22:18:39.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:18:39.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:18:39.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:18:39.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.254.60:44023
22:18:39.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:18:39.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
22:18:39.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:18:39.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37345/jwks.json
22:18:39.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
22:18:39.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
22:18:39.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
22:18:39.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:18:39.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:18:39.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-2/data/info.pb
22:18:39.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.254.60
22:18:39.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41665
22:18:39.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
22:18:39.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:18:39.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
22:18:39.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
22:18:39.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:18:39.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-2/logs
22:18:39.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:18:39.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
22:18:39.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:18:39.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
22:18:39.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
22:18:39.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
22:18:39.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
22:18:39.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 21:22:24 UTC on ccycloud.quasar-wysbne.root.comops.site
22:18:39.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676318
22:18:39.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
22:18:39.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:39.402580 23950 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
22:18:39.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:39.403869 23950 file_cache.cc:493] Constructed file cache file cache with capacity 419430
22:18:39.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:39.423079 23973 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:18:40.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:40.487589 23979 raft_consensus.cc:482] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
22:18:40.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:40.488118 23979 raft_consensus.cc:504] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "343dd293809c42459b96126818d74182" member_type: VOTER last_known_addr { host: "127.22.254.62" port: 37215 } } peers { permanent_uuid: "61f1a35769dd4952861645014e55e229" member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } } peers { permanent_uuid: "0cb22a9de93b4bfb93a644663cb469af" member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } }
22:18:40.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:40.490911 23979 leader_election.cc:288] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 61f1a35769dd4952861645014e55e229 (127.22.254.61:36153), 0cb22a9de93b4bfb93a644663cb469af (127.22.254.60:44023)
22:18:40.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:40.501211 23900 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.254.60:44023: connect: Connection refused (error 111)
22:18:40.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:40.510807 23900 leader_election.cc:334] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 0cb22a9de93b4bfb93a644663cb469af (127.22.254.60:44023): Network error: Client connection negotiation failed: client connection to 127.22.254.60:44023: connect: Connection refused (error 111)
22:18:40.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:40.512532 23897 leader_election.cc:334] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 61f1a35769dd4952861645014e55e229 (127.22.254.61:36153): Network error: Client connection negotiation failed: client connection to 127.22.254.61:36153: connect: Connection refused (error 111)
22:18:40.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:40.512934 23897 leader_election.cc:302] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 343dd293809c42459b96126818d74182; no voters: 0cb22a9de93b4bfb93a644663cb469af, 61f1a35769dd4952861645014e55e229
22:18:40.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:40.514364 23979 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
22:18:40.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:39.424571 23950 server_base.cc:1034] running on GCE node
22:18:40.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:39.428474 23975 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:18:40.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:39.423494 23972 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:18:40.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:40.689445 23950 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
22:18:40.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:40.692627 23950 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
22:18:40.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:40.694177 23950 hybrid_clock.cc:648] HybridClock initialized: now 1790547520694113 us; error 82 us; skew 500 ppm
22:18:40.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:40.695024 23950 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
22:18:40.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:18:40 dist-test-slave-qzth krb5kdc[23555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547520, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.254.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
22:18:40.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:40.700670 23950 init.cc:377] Logged in from keytab as kudu/127.22.254.60@KRBTEST.COM (short username kudu)
22:18:40.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:40.706961 23950 webserver.cc:460] Webserver started at http://127.22.254.60:41665/ using document root <none> and password file <none>
22:18:40.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:40.708098 23950 fs_manager.cc:359] Metadata directory not provided
22:18:40.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:40.708361 23950 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:18:40.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:40.718489 23950 fs_manager.cc:709] Time spent opening directory manager: real 0.007s user 0.006s sys 0.001s
22:18:40.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:40.724840 23988 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
22:18:40.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:40.726012 23950 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.006s sys 0.001s
22:18:40.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:40.726362 23950 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-2/data,/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-2/wal
22:18:40.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0cb22a9de93b4bfb93a644663cb469af"
22:18:40.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:18:33 on dist-test-slave-qzth"
22:18:40.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:40.728659 23950 fs_report.cc:383] FS layout report
22:18:40.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
22:18:40.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-2/wal
22:18:40.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-2/wal
22:18:40.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-2/data/data
22:18:40.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
22:18:40.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
22:18:40.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
22:18:40.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
22:18:40.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
22:18:40.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
22:18:40.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
22:18:40.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
22:18:40.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
22:18:40.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
22:18:40.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
22:18:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:40.767304 23950 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
22:18:40.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:40.774772 23950 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
22:18:40.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:40.775329 23950 kserver.cc:163] Server-wide thread pool size limit: 3276
22:18:40.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:40.856594 23950 rpc_server.cc:307] RPC server started. Bound to: 127.22.254.60:44023
22:18:40.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:40.856729 24039 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.254.60:44023 every 8 connection(s)
22:18:40.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:40.860231 23950 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-2/data/info.pb
22:18:40.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:40.870654 23544 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu as pid 23950
22:18:40.871 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
22:18:40.871 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.22.254.61:36153
22:18:40.871 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
22:18:40.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:40.872030 24040 sys_catalog.cc:263] Verifying existing consensus state
22:18:40.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:40.873267 23544 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:18:40.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu
22:18:40.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-1/wal
22:18:40.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-1/data
22:18:40.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
22:18:40.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
22:18:40.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
22:18:40.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:18:40.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:18:40.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:18:40.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-1/logs
22:18:40.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-1/data/info.pb
22:18:40.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:18:40.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
22:18:40.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
22:18:40.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
22:18:40.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
22:18:40.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:18:40.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:18:40.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:18:40.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
22:18:40.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
22:18:40.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:18:40.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:18:40.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.254.61:36153
22:18:40.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.254.61
22:18:40.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33883
22:18:40.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/krb5kdc/kudu_127.22.254.61.keytab
22:18:40.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.254.61
22:18:40.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
22:18:40.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
22:18:40.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
22:18:40.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.254.20:45025
22:18:40.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:18:40.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:18:40.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:18:40.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:18:40.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.254.62:37215,127.22.254.61:36153,127.22.254.60:44023
22:18:40.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:18:40.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37345/jwks.json
22:18:40.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
22:18:40.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:40.879983 24040 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af: Bootstrap starting.
22:18:40.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:40.925269 24040 log.cc:824] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af: Log is configured to *not* fsync() on all Append() calls
22:18:40.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:40.944941 24040 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
22:18:40.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:40.946086 24040 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af: Bootstrap complete.
22:18:40.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:40.980947 24040 raft_consensus.cc:348] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "343dd293809c42459b96126818d74182" member_type: VOTER last_known_addr { host: "127.22.254.62" port: 37215 } } peers { permanent_uuid: "61f1a35769dd4952861645014e55e229" member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } } peers { permanent_uuid: "0cb22a9de93b4bfb93a644663cb469af" member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } }
22:18:40.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:40.984335 24040 raft_consensus.cc:729] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0cb22a9de93b4bfb93a644663cb469af, State: Initialized, Role: FOLLOWER
22:18:40.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:40.985208 24040 consensus_queue.cc:260] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "343dd293809c42459b96126818d74182" member_type: VOTER last_known_addr { host: "127.22.254.62" port: 37215 } } peers { permanent_uuid: "61f1a35769dd4952861645014e55e229" member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } } peers { permanent_uuid: "0cb22a9de93b4bfb93a644663cb469af" member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } }
22:18:40.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:40.989162 24044 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "343dd293809c42459b96126818d74182" member_type: VOTER last_known_addr { host: "127.22.254.62" port: 37215 } } peers { permanent_uuid: "61f1a35769dd4952861645014e55e229" member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } } peers { permanent_uuid: "0cb22a9de93b4bfb93a644663cb469af" member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } } }
22:18:40.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:40.990393 24044 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af [sys.catalog]: This master's current role is: FOLLOWER
22:18:40.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:40.990831 24040 sys_catalog.cc:564] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af [sys.catalog]: configured and running, proceeding with master startup.
22:18:41.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:41.046998 24055 catalog_manager.cc:1259] Loaded cluster ID: 3feed9418f774d00b5c787fb3c6c3065
22:18:41.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:41.047561 24055 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af: loading cluster ID for follower catalog manager: success
22:18:41.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:41.058029 24055 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af: acquiring CA information for follower catalog manager: success
22:18:41.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:41.069200 24055 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
22:18:41.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:18:41 dist-test-slave-qzth krb5kdc[23555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547520, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.254.60@KRBTEST.COM for kudu/127.22.254.62@KRBTEST.COM
22:18:41.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:41.094691 23950 master_runner.cc:186] Error getting master registration for 127.22.254.61:36153: Network error: Client connection negotiation failed: client connection to 127.22.254.61:36153: connect: Connection refused (error 111),
22:18:41.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:41.122848 23950 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
22:18:41.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:41.255733 24042 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
22:18:41.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:41.256327 24042 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
22:18:41.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:41.256959 24042 flags.cc:425] Enabled unsafe flag: --never_fsync=true
22:18:41.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:41.293875 24042 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
22:18:41.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:41.294132 24042 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
22:18:41.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:41.294508 24042 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
22:18:41.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:41.294785 24042 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
22:18:41.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:41.295046 24042 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
22:18:41.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:41.295375 24042 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
22:18:41.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:41.295574 24042 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37345/jwks.json
22:18:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:41.333171 24042 master_runner.cc:386] Master server non-default flags:
22:18:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:18:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.254.20:45025
22:18:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:18:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:18:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-1/data
22:18:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-1/wal
22:18:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
22:18:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:18:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.254.62:37215,127.22.254.61:36153,127.22.254.60:44023
22:18:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/krb5kdc/kudu_127.22.254.61.keytab
22:18:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.254.61
22:18:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:18:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:18:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:18:41.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.254.61:36153
22:18:41.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:18:41.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
22:18:41.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:18:41.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37345/jwks.json
22:18:41.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
22:18:41.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
22:18:41.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
22:18:41.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:18:41.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:18:41.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-1/data/info.pb
22:18:41.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.254.61
22:18:41.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33883
22:18:41.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
22:18:41.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:18:41.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
22:18:41.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
22:18:41.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:18:41.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-1/logs
22:18:41.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:18:41.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
22:18:41.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:18:41.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
22:18:41.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
22:18:41.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
22:18:41.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
22:18:41.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 21:22:24 UTC on ccycloud.quasar-wysbne.root.comops.site
22:18:41.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676318
22:18:41.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
22:18:41.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:41.334585 24042 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
22:18:41.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:41.335726 24042 file_cache.cc:493] Constructed file cache file cache with capacity 419430
22:18:41.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:41.365700 24065 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:18:42.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.019737 24070 raft_consensus.cc:482] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
22:18:42.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.020162 24070 raft_consensus.cc:504] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "343dd293809c42459b96126818d74182" member_type: VOTER last_known_addr { host: "127.22.254.62" port: 37215 } } peers { permanent_uuid: "61f1a35769dd4952861645014e55e229" member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } } peers { permanent_uuid: "0cb22a9de93b4bfb93a644663cb469af" member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } }
22:18:42.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.022073 24070 leader_election.cc:288] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 61f1a35769dd4952861645014e55e229 (127.22.254.61:36153), 0cb22a9de93b4bfb93a644663cb469af (127.22.254.60:44023)
22:18:42.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:42.060580 23897 leader_election.cc:334] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 61f1a35769dd4952861645014e55e229 (127.22.254.61:36153): Network error: Client connection negotiation failed: client connection to 127.22.254.61:36153: connect: Connection refused (error 111)
22:18:42.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.063434 24015 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "343dd293809c42459b96126818d74182" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "0cb22a9de93b4bfb93a644663cb469af" is_pre_election: true
22:18:42.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.064205 24015 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 343dd293809c42459b96126818d74182 in term 1.
22:18:42.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.066319 23900 leader_election.cc:302] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 0cb22a9de93b4bfb93a644663cb469af, 343dd293809c42459b96126818d74182; no voters: 61f1a35769dd4952861645014e55e229
22:18:42.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.067371 24070 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [term 1 FOLLOWER]: Leader pre-election won for term 2
22:18:42.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.067667 24070 raft_consensus.cc:482] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
22:18:42.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.067955 24070 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [term 1 FOLLOWER]: Advancing to term 2
22:18:42.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.077613 24070 raft_consensus.cc:504] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "343dd293809c42459b96126818d74182" member_type: VOTER last_known_addr { host: "127.22.254.62" port: 37215 } } peers { permanent_uuid: "61f1a35769dd4952861645014e55e229" member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } } peers { permanent_uuid: "0cb22a9de93b4bfb93a644663cb469af" member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } }
22:18:42.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.080380 24015 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "343dd293809c42459b96126818d74182" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "0cb22a9de93b4bfb93a644663cb469af"
22:18:42.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.080885 24015 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af [term 1 FOLLOWER]: Advancing to term 2
22:18:42.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:42.086525 23897 leader_election.cc:334] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 61f1a35769dd4952861645014e55e229 (127.22.254.61:36153): Network error: Client connection negotiation failed: client connection to 127.22.254.61:36153: connect: Connection refused (error 111)
22:18:42.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.087785 24070 leader_election.cc:288] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [CANDIDATE]: Term 2 election: Requested vote from peers 61f1a35769dd4952861645014e55e229 (127.22.254.61:36153), 0cb22a9de93b4bfb93a644663cb469af (127.22.254.60:44023)
22:18:42.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.090047 24015 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 343dd293809c42459b96126818d74182 in term 2.
22:18:42.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.091048 23900 leader_election.cc:302] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 0cb22a9de93b4bfb93a644663cb469af, 343dd293809c42459b96126818d74182; no voters: 61f1a35769dd4952861645014e55e229
22:18:42.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.092502 24070 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [term 2 FOLLOWER]: Leader election won for term 2
22:18:42.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.094477 24070 raft_consensus.cc:686] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [term 2 LEADER]: Becoming Leader. State: Replica: 343dd293809c42459b96126818d74182, State: Running, Role: LEADER
22:18:42.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.095321 24070 consensus_queue.cc:237] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "343dd293809c42459b96126818d74182" member_type: VOTER last_known_addr { host: "127.22.254.62" port: 37215 } } peers { permanent_uuid: "61f1a35769dd4952861645014e55e229" member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } } peers { permanent_uuid: "0cb22a9de93b4bfb93a644663cb469af" member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } }
22:18:42.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.113708 24075 sys_catalog.cc:455] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 343dd293809c42459b96126818d74182. Latest consensus state: current_term: 2 leader_uuid: "343dd293809c42459b96126818d74182" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "343dd293809c42459b96126818d74182" member_type: VOTER last_known_addr { host: "127.22.254.62" port: 37215 } } peers { permanent_uuid: "61f1a35769dd4952861645014e55e229" member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } } peers { permanent_uuid: "0cb22a9de93b4bfb93a644663cb469af" member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } } }
22:18:42.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.115653 24075 sys_catalog.cc:458] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [sys.catalog]: This master's current role is: LEADER
22:18:42.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.118331 24077 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
22:18:42.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.130259 24077 catalog_manager.cc:1484] Initializing Kudu cluster ID...
22:18:42.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.135842 24077 catalog_manager.cc:1259] Loaded cluster ID: 3feed9418f774d00b5c787fb3c6c3065
22:18:42.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.136189 24077 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
22:18:42.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.144140 24077 catalog_manager.cc:1504] Loading token signing keys...
22:18:42.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.149693 24077 catalog_manager.cc:5879] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182: Loaded TSK: 0
22:18:42.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.151160 24077 catalog_manager.cc:1514] Initializing in-progress tserver states...
22:18:42.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.536474 24015 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af [term 2 FOLLOWER]: Refusing update from remote peer 343dd293809c42459b96126818d74182: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
22:18:42.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.537992 24075 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0cb22a9de93b4bfb93a644663cb469af" member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
22:18:42.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.552425 24078 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af [sys.catalog]: SysCatalogTable state changed. Reason: New leader 343dd293809c42459b96126818d74182. Latest consensus state: current_term: 2 leader_uuid: "343dd293809c42459b96126818d74182" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "343dd293809c42459b96126818d74182" member_type: VOTER last_known_addr { host: "127.22.254.62" port: 37215 } } peers { permanent_uuid: "61f1a35769dd4952861645014e55e229" member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } } peers { permanent_uuid: "0cb22a9de93b4bfb93a644663cb469af" member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } } }
22:18:42.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.553668 24078 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af [sys.catalog]: This master's current role is: FOLLOWER
22:18:42.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.558013 24078 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "343dd293809c42459b96126818d74182" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "343dd293809c42459b96126818d74182" member_type: VOTER last_known_addr { host: "127.22.254.62" port: 37215 } } peers { permanent_uuid: "61f1a35769dd4952861645014e55e229" member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } } peers { permanent_uuid: "0cb22a9de93b4bfb93a644663cb469af" member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } } }
22:18:42.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.558478 24078 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0cb22a9de93b4bfb93a644663cb469af [sys.catalog]: This master's current role is: FOLLOWER
22:18:42.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.571614 24070 sys_catalog.cc:455] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "343dd293809c42459b96126818d74182" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "343dd293809c42459b96126818d74182" member_type: VOTER last_known_addr { host: "127.22.254.62" port: 37215 } } peers { permanent_uuid: "61f1a35769dd4952861645014e55e229" member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } } peers { permanent_uuid: "0cb22a9de93b4bfb93a644663cb469af" member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } } }
22:18:42.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.572317 24070 sys_catalog.cc:458] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [sys.catalog]: This master's current role is: LEADER
22:18:42.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:42.575328 23897 consensus_peers.cc:487] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 -> Peer 61f1a35769dd4952861645014e55e229 (127.22.254.61:36153): Couldn't send request to peer 61f1a35769dd4952861645014e55e229. Status: Network error: Client connection negotiation failed: client connection to 127.22.254.61:36153: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
22:18:42.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:41.366888 24064 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:18:42.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:41.365713 24067 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:18:42.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:42.673234 24066 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
22:18:42.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.673761 24042 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
22:18:42.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.678012 24042 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
22:18:42.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.682361 24042 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
22:18:42.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.683771 24042 hybrid_clock.cc:648] HybridClock initialized: now 1790547522683698 us; error 73 us; skew 500 ppm
22:18:42.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.685135 24042 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
22:18:42.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:18:42 dist-test-slave-qzth krb5kdc[23555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547522, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.254.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
22:18:42.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.702387 24042 init.cc:377] Logged in from keytab as kudu/127.22.254.61@KRBTEST.COM (short username kudu)
22:18:42.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.708070 24042 webserver.cc:460] Webserver started at http://127.22.254.61:33883/ using document root <none> and password file <none>
22:18:42.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.709139 24042 fs_manager.cc:359] Metadata directory not provided
22:18:42.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.709327 24042 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:18:42.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.717550 24042 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.003s sys 0.004s
22:18:42.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.722671 24087 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
22:18:42.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.723891 24042 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
22:18:42.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.724282 24042 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-1/data,/tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-1/wal
22:18:42.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "61f1a35769dd4952861645014e55e229"
22:18:42.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 22:18:31 on dist-test-slave-qzth"
22:18:42.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.727418 24042 fs_report.cc:383] FS layout report
22:18:42.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
22:18:42.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-1/wal
22:18:42.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-1/wal
22:18:42.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-1/data/data
22:18:42.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
22:18:42.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
22:18:42.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
22:18:42.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
22:18:42.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
22:18:42.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
22:18:42.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
22:18:42.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
22:18:42.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
22:18:42.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
22:18:42.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
22:18:42.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.785552 24042 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
22:18:42.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.791982 24042 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
22:18:42.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.792622 24042 kserver.cc:163] Server-wide thread pool size limit: 3276
22:18:42.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.883479 24042 rpc_server.cc:307] RPC server started. Bound to: 127.22.254.61:36153
22:18:42.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.883574 24138 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.254.61:36153 every 8 connection(s)
22:18:42.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.887481 24042 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskyO6VHJ/test-tmp/mini-kudu-cluster6692882828770984839/master-1/data/info.pb
22:18:42.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.889991 23544 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu as pid 24042
22:18:42.890 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
22:18:42.913 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.22.254.62:37215(127.22.254.62:37215)] connecting to peer
22:18:42.918 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.22.254.60:44023(127.22.254.60:44023)] connecting to peer
22:18:42.919 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.22.254.61:36153(127.22.254.61:36153)] connecting to peer
22:18:42.945 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.22.254.62:37215(127.22.254.62:37215)] Successfully connected to peer
22:18:42.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.925799 24140 sys_catalog.cc:263] Verifying existing consensus state
22:18:42.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:42.933171 24140 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 61f1a35769dd4952861645014e55e229: Bootstrap starting.
22:18:42.948 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.22.254.60:44023(127.22.254.60:44023)] Successfully connected to peer
22:18:42.952 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.22.254.61:36153(127.22.254.61:36153)] Successfully connected to peer
22:18:42.956 [WARN - kudu-nio-1] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
22:18:42.958 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.22.254.60
22:18:42.970 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.22.254.62
22:18:42.996 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.22.254.61
22:18:43.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:43.009400 24140 log.cc:824] T 00000000000000000000000000000000 P 61f1a35769dd4952861645014e55e229: Log is configured to *not* fsync() on all Append() calls
22:18:43.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:18:43 dist-test-slave-qzth krb5kdc[23555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547513, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.22.254.60@KRBTEST.COM
22:18:43.028 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.254.62:37215(127.22.254.62:37215)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
22:18:43.029 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.62:37215: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
22:18:43.029 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x8bee98d2, L:/127.0.0.1:60206 - R:/127.22.254.60:44023] using SASL/GSSAPI
22:18:43.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:43.031801 24146 negotiation.cc:337] Failed RPC negotiation. Trace:
22:18:43.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:42.939449 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:55736 (local address 127.22.254.62:37215)
22:18:43.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:42.969038 (+ 29589us) server_negotiation.cc:206] Beginning negotiation
22:18:43.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:42.969059 (+ 21us) server_negotiation.cc:399] Waiting for connection header
22:18:43.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:42.969102 (+ 43us) server_negotiation.cc:407] Connection header received
22:18:43.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:42.969237 (+ 135us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:18:43.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:42.969259 (+ 22us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:18:43.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:42.969455 (+ 196us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:18:43.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:42.973881 (+ 4426us) server_negotiation.cc:226] Negotiated authn=SASL
22:18:43.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:42.981001 (+ 7120us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
22:18:43.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:42.983797 (+ 2796us) server_negotiation.cc:377] Sending TLS_HANDSHAKE NegotiatePB response
22:18:43.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:42.994803 (+ 11006us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
22:18:43.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:42.995671 (+ 868us) server_negotiation.cc:377] Sending TLS_HANDSHAKE NegotiatePB response
22:18:43.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:42.995936 (+ 265us) server_negotiation.cc:662] Negotiated TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 TLSv1.2 Kx=ECDH Au=RSA Enc=AESGCM(128) Mac=AEAD
22:18:43.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.030174 (+ 34238us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:55736: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:55736): Cannot send after transport endpoint shutdown (error 108)
22:18:43.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":29185,"thread_start_us":28887,"threads_started":1}
22:18:43.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:43.032578 24140 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 61f1a35769dd4952861645014e55e229: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
22:18:43.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:43.033421 24140 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 61f1a35769dd4952861645014e55e229: Bootstrap complete.
22:18:43.034 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.22.254.62:37215(127.22.254.62:37215)] handling channelInactive
22:18:43.036 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.254.61:36153(127.22.254.61:36153)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
22:18:43.036 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.61:36153: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
22:18:43.036 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153; will retry
22:18:43.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:43.039693 24147 negotiation.cc:337] Failed RPC negotiation. Trace:
22:18:43.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:42.939866 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:44414 (local address 127.22.254.61:36153)
22:18:43.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:42.960390 (+ 20524us) server_negotiation.cc:206] Beginning negotiation
22:18:43.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:42.960437 (+ 47us) server_negotiation.cc:399] Waiting for connection header
22:18:43.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:42.960518 (+ 81us) server_negotiation.cc:407] Connection header received
22:18:43.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:42.989241 (+ 28723us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:18:43.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:42.989294 (+ 53us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:18:43.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:42.989736 (+ 442us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:18:43.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:42.990164 (+ 428us) server_negotiation.cc:226] Negotiated authn=SASL
22:18:43.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.014856 (+ 24692us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
22:18:43.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.017331 (+ 2475us) server_negotiation.cc:377] Sending TLS_HANDSHAKE NegotiatePB response
22:18:43.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.024877 (+ 7546us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
22:18:43.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.026080 (+ 1203us) server_negotiation.cc:377] Sending TLS_HANDSHAKE NegotiatePB response
22:18:43.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.026391 (+ 311us) server_negotiation.cc:662] Negotiated TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 TLSv1.2 Kx=ECDH Au=RSA Enc=AESGCM(128) Mac=AEAD
22:18:43.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.037872 (+ 11481us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:44414: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:44414): Cannot send after transport endpoint shutdown (error 108)
22:18:43.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":19889,"thread_start_us":19499,"threads_started":1}
22:18:43.054 [DEBUG - pool-70-thread-1] (Connection.java:748) [peer master-127.22.254.62:37215(127.22.254.62:37215)] connecting to peer
22:18:43.055 [DEBUG - pool-70-thread-1] (Connection.java:748) [peer master-127.22.254.61:36153(127.22.254.61:36153)] connecting to peer
22:18:43.056 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.22.254.62:37215(127.22.254.62:37215)] Successfully connected to peer
22:18:43.058 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.254.62:37215(127.22.254.62:37215)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:43.059 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.62:37215: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:43.059 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.22.254.62:37215(127.22.254.62:37215)] handling channelInactive
22:18:43.063 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.22.254.61:36153(127.22.254.61:36153)] handling channelInactive
22:18:43.064 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.22.254.61:36153(127.22.254.61:36153)] Successfully connected to peer
22:18:43.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:43.062813 24140 raft_consensus.cc:348] T 00000000000000000000000000000000 P 61f1a35769dd4952861645014e55e229 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "343dd293809c42459b96126818d74182" member_type: VOTER last_known_addr { host: "127.22.254.62" port: 37215 } } peers { permanent_uuid: "61f1a35769dd4952861645014e55e229" member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } } peers { permanent_uuid: "0cb22a9de93b4bfb93a644663cb469af" member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } }
22:18:43.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:43.066041 24140 raft_consensus.cc:729] T 00000000000000000000000000000000 P 61f1a35769dd4952861645014e55e229 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 61f1a35769dd4952861645014e55e229, State: Initialized, Role: FOLLOWER
22:18:43.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:43.066900 24140 consensus_queue.cc:260] T 00000000000000000000000000000000 P 61f1a35769dd4952861645014e55e229 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "343dd293809c42459b96126818d74182" member_type: VOTER last_known_addr { host: "127.22.254.62" port: 37215 } } peers { permanent_uuid: "61f1a35769dd4952861645014e55e229" member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } } peers { permanent_uuid: "0cb22a9de93b4bfb93a644663cb469af" member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } }
22:18:43.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:43.065038 24146 negotiation.cc:337] Failed RPC negotiation. Trace:
22:18:43.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.056168 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:55746 (local address 127.22.254.62:37215)
22:18:43.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.057552 (+ 1384us) server_negotiation.cc:206] Beginning negotiation
22:18:43.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.057575 (+ 23us) server_negotiation.cc:399] Waiting for connection header
22:18:43.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.057630 (+ 55us) server_negotiation.cc:407] Connection header received
22:18:43.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.057738 (+ 108us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:18:43.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.057760 (+ 22us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:18:43.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.057973 (+ 213us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:18:43.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.058156 (+ 183us) server_negotiation.cc:226] Negotiated authn=SASL
22:18:43.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.064057 (+ 5901us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:55746: BlockingRecv error: recv got EOF from 127.0.0.1:55746 (error 108)
22:18:43.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":1036}
22:18:43.070 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.254.61:36153(127.22.254.61:36153)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:43.071 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.61:36153: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:43.071 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153; will retry
22:18:43.071 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.22.254.61:36153(127.22.254.61:36153)] handling channelInactive
22:18:43.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:43.073976 24149 negotiation.cc:337] Failed RPC negotiation. Trace:
22:18:43.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.066322 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:44434 (local address 127.22.254.61:36153)
22:18:43.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.069707 (+ 3385us) server_negotiation.cc:206] Beginning negotiation
22:18:43.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.069737 (+ 30us) server_negotiation.cc:399] Waiting for connection header
22:18:43.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.069778 (+ 41us) server_negotiation.cc:407] Connection header received
22:18:43.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.069950 (+ 172us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:18:43.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.069974 (+ 24us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:18:43.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.070173 (+ 199us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:18:43.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.071877 (+ 1704us) server_negotiation.cc:226] Negotiated authn=SASL
22:18:43.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.072680 (+ 803us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:44434: BlockingRecv error: recv got EOF from 127.0.0.1:44434 (error 108)
22:18:43.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":2982,"thread_start_us":5887,"threads_started":1}
22:18:43.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:43.074958 24150 sys_catalog.cc:455] T 00000000000000000000000000000000 P 61f1a35769dd4952861645014e55e229 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "343dd293809c42459b96126818d74182" member_type: VOTER last_known_addr { host: "127.22.254.62" port: 37215 } } peers { permanent_uuid: "61f1a35769dd4952861645014e55e229" member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } } peers { permanent_uuid: "0cb22a9de93b4bfb93a644663cb469af" member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } } }
22:18:43.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:43.075726 24150 sys_catalog.cc:458] T 00000000000000000000000000000000 P 61f1a35769dd4952861645014e55e229 [sys.catalog]: This master's current role is: FOLLOWER
22:18:43.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:43.076116 24140 sys_catalog.cc:564] T 00000000000000000000000000000000 P 61f1a35769dd4952861645014e55e229 [sys.catalog]: configured and running, proceeding with master startup.
22:18:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:18:43 dist-test-slave-qzth krb5kdc[23555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547518, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.254.62@KRBTEST.COM for kudu/127.22.254.61@KRBTEST.COM
22:18:43.093 [DEBUG - pool-70-thread-1] (Connection.java:748) [peer master-127.22.254.62:37215(127.22.254.62:37215)] connecting to peer
22:18:43.094 [DEBUG - pool-70-thread-1] (Connection.java:748) [peer master-127.22.254.61:36153(127.22.254.61:36153)] connecting to peer
22:18:43.095 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.22.254.61:36153(127.22.254.61:36153)] Successfully connected to peer
22:18:43.097 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.254.61:36153(127.22.254.61:36153)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:43.097 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.61:36153: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:43.097 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.22.254.61:36153(127.22.254.61:36153)] handling channelInactive
22:18:43.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:43.099794 24114 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 61f1a35769dd4952861645014e55e229 [term 1 FOLLOWER]: Advancing to term 2
22:18:43.103 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.22.254.62:37215(127.22.254.62:37215)] Successfully connected to peer
22:18:43.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:43.104971 24149 negotiation.cc:337] Failed RPC negotiation. Trace:
22:18:43.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.095589 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:44446 (local address 127.22.254.61:36153)
22:18:43.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.096230 (+ 641us) server_negotiation.cc:206] Beginning negotiation
22:18:43.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.096262 (+ 32us) server_negotiation.cc:399] Waiting for connection header
22:18:43.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.096311 (+ 49us) server_negotiation.cc:407] Connection header received
22:18:43.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.096448 (+ 137us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:18:43.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.096484 (+ 36us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:18:43.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.096734 (+ 250us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:18:43.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.096931 (+ 197us) server_negotiation.cc:226] Negotiated authn=SASL
22:18:43.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.104037 (+ 7106us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:44446: BlockingRecv error: recv got EOF from 127.0.0.1:44446 (error 108)
22:18:43.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":140}
22:18:43.107 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.254.62:37215(127.22.254.62:37215)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:43.107 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.62:37215: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:43.107 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153; will retry
22:18:43.108 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.22.254.62:37215(127.22.254.62:37215)] handling channelInactive
22:18:43.113 [DEBUG - pool-70-thread-1] (Connection.java:748) [peer master-127.22.254.62:37215(127.22.254.62:37215)] connecting to peer
22:18:43.113 [DEBUG - pool-70-thread-1] (Connection.java:748) [peer master-127.22.254.61:36153(127.22.254.61:36153)] connecting to peer
22:18:43.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:43.112684 24146 negotiation.cc:337] Failed RPC negotiation. Trace:
22:18:43.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.102298 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:55754 (local address 127.22.254.62:37215)
22:18:43.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.105595 (+ 3297us) server_negotiation.cc:206] Beginning negotiation
22:18:43.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.105623 (+ 28us) server_negotiation.cc:399] Waiting for connection header
22:18:43.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.105665 (+ 42us) server_negotiation.cc:407] Connection header received
22:18:43.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.105906 (+ 241us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:18:43.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.105937 (+ 31us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:18:43.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.106219 (+ 282us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:18:43.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.106378 (+ 159us) server_negotiation.cc:226] Negotiated authn=SASL
22:18:43.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.108524 (+ 2146us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:55754: BlockingRecv error: recv got EOF from 127.0.0.1:55754 (error 108)
22:18:43.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":2918}
22:18:43.115 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.22.254.61:36153(127.22.254.61:36153)] Successfully connected to peer
22:18:43.117 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.254.61:36153(127.22.254.61:36153)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:43.117 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.22.254.62:37215(127.22.254.62:37215)] Successfully connected to peer
22:18:43.117 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.61:36153: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:43.117 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.22.254.61:36153(127.22.254.61:36153)] handling channelInactive
22:18:43.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:43.118636 24149 negotiation.cc:337] Failed RPC negotiation. Trace:
22:18:43.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.115853 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:44462 (local address 127.22.254.61:36153)
22:18:43.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.116270 (+ 417us) server_negotiation.cc:206] Beginning negotiation
22:18:43.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.116298 (+ 28us) server_negotiation.cc:399] Waiting for connection header
22:18:43.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.116338 (+ 40us) server_negotiation.cc:407] Connection header received
22:18:43.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.116455 (+ 117us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:18:43.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.116478 (+ 23us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:18:43.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.116674 (+ 196us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:18:43.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.116830 (+ 156us) server_negotiation.cc:226] Negotiated authn=SASL
22:18:43.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.117758 (+ 928us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:44462: BlockingRecv error: recv got EOF from 127.0.0.1:44462 (error 108)
22:18:43.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":64}
22:18:43.123 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.254.62:37215(127.22.254.62:37215)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:43.124 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.62:37215: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:43.124 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153; will retry
22:18:43.124 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.22.254.62:37215(127.22.254.62:37215)] handling channelInactive
22:18:43.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:43.126542 24146 negotiation.cc:337] Failed RPC negotiation. Trace:
22:18:43.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.113959 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:55766 (local address 127.22.254.62:37215)
22:18:43.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.122208 (+ 8249us) server_negotiation.cc:206] Beginning negotiation
22:18:43.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.122244 (+ 36us) server_negotiation.cc:399] Waiting for connection header
22:18:43.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.122300 (+ 56us) server_negotiation.cc:407] Connection header received
22:18:43.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.122419 (+ 119us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:18:43.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.122443 (+ 24us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:18:43.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.122720 (+ 277us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:18:43.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.122913 (+ 193us) server_negotiation.cc:226] Negotiated authn=SASL
22:18:43.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.125357 (+ 2444us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:55766: BlockingRecv error: recv got EOF from 127.0.0.1:55766 (error 108)
22:18:43.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":7629}
22:18:43.133 [DEBUG - pool-70-thread-1] (Connection.java:748) [peer master-127.22.254.62:37215(127.22.254.62:37215)] connecting to peer
22:18:43.134 [DEBUG - pool-70-thread-1] (Connection.java:748) [peer master-127.22.254.61:36153(127.22.254.61:36153)] connecting to peer
22:18:43.135 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.22.254.62:37215(127.22.254.62:37215)] Successfully connected to peer
22:18:43.136 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.22.254.61:36153(127.22.254.61:36153)] Successfully connected to peer
22:18:43.139 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.254.62:37215(127.22.254.62:37215)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:43.139 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.62:37215: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:43.139 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.22.254.62:37215(127.22.254.62:37215)] handling channelInactive
22:18:43.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:43.141178 24146 negotiation.cc:337] Failed RPC negotiation. Trace:
22:18:43.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.137273 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:55772 (local address 127.22.254.62:37215)
22:18:43.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.137737 (+ 464us) server_negotiation.cc:206] Beginning negotiation
22:18:43.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.137760 (+ 23us) server_negotiation.cc:399] Waiting for connection header
22:18:43.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.137803 (+ 43us) server_negotiation.cc:407] Connection header received
22:18:43.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.137940 (+ 137us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:18:43.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.137962 (+ 22us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:18:43.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.138181 (+ 219us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:18:43.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.138351 (+ 170us) server_negotiation.cc:226] Negotiated authn=SASL
22:18:43.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.140322 (+ 1971us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:55772: BlockingRecv error: recv got EOF from 127.0.0.1:55772 (error 108)
22:18:43.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":110}
22:18:43.142 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.254.61:36153(127.22.254.61:36153)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:43.142 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.61:36153: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:43.142 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153; will retry
22:18:43.143 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.22.254.61:36153(127.22.254.61:36153)] handling channelInactive
22:18:43.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:43.144603 24149 negotiation.cc:337] Failed RPC negotiation. Trace:
22:18:43.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.140491 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:44478 (local address 127.22.254.61:36153)
22:18:43.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.140981 (+ 490us) server_negotiation.cc:206] Beginning negotiation
22:18:43.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.141005 (+ 24us) server_negotiation.cc:399] Waiting for connection header
22:18:43.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.141056 (+ 51us) server_negotiation.cc:407] Connection header received
22:18:43.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.141172 (+ 116us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:18:43.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.141196 (+ 24us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:18:43.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.141383 (+ 187us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:18:43.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.141544 (+ 161us) server_negotiation.cc:226] Negotiated authn=SASL
22:18:43.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.143697 (+ 2153us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:44478: BlockingRecv error: recv got EOF from 127.0.0.1:44478 (error 108)
22:18:43.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":115}
22:18:43.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:43.134951 24163 sys_catalog.cc:455] T 00000000000000000000000000000000 P 61f1a35769dd4952861645014e55e229 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "343dd293809c42459b96126818d74182" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "343dd293809c42459b96126818d74182" member_type: VOTER last_known_addr { host: "127.22.254.62" port: 37215 } } peers { permanent_uuid: "61f1a35769dd4952861645014e55e229" member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } } peers { permanent_uuid: "0cb22a9de93b4bfb93a644663cb469af" member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } } }
22:18:43.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:43.145774 24150 sys_catalog.cc:455] T 00000000000000000000000000000000 P 61f1a35769dd4952861645014e55e229 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 343dd293809c42459b96126818d74182. Latest consensus state: current_term: 2 leader_uuid: "343dd293809c42459b96126818d74182" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "343dd293809c42459b96126818d74182" member_type: VOTER last_known_addr { host: "127.22.254.62" port: 37215 } } peers { permanent_uuid: "61f1a35769dd4952861645014e55e229" member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } } peers { permanent_uuid: "0cb22a9de93b4bfb93a644663cb469af" member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } } }
22:18:43.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:43.146462 24150 sys_catalog.cc:458] T 00000000000000000000000000000000 P 61f1a35769dd4952861645014e55e229 [sys.catalog]: This master's current role is: FOLLOWER
22:18:43.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:43.145242 24163 sys_catalog.cc:458] T 00000000000000000000000000000000 P 61f1a35769dd4952861645014e55e229 [sys.catalog]: This master's current role is: FOLLOWER
22:18:43.153 [DEBUG - pool-70-thread-1] (Connection.java:748) [peer master-127.22.254.62:37215(127.22.254.62:37215)] connecting to peer
22:18:43.153 [DEBUG - pool-70-thread-1] (Connection.java:748) [peer master-127.22.254.61:36153(127.22.254.61:36153)] connecting to peer
22:18:43.156 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.22.254.62:37215(127.22.254.62:37215)] Successfully connected to peer
22:18:43.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:18:43 dist-test-slave-qzth krb5kdc[23555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547522, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.254.61@KRBTEST.COM for kudu/127.22.254.62@KRBTEST.COM
22:18:43.165 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.22.254.61:36153(127.22.254.61:36153)] Successfully connected to peer
22:18:43.168 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.254.61:36153(127.22.254.61:36153)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:43.169 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.61:36153: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:43.171 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.254.62:37215(127.22.254.62:37215)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:43.171 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.62:37215: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:43.171 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153; will retry
22:18:43.172 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.22.254.62:37215(127.22.254.62:37215)] handling channelInactive
22:18:43.173 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.22.254.61:36153(127.22.254.61:36153)] handling channelInactive
22:18:43.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:43.173307 24146 negotiation.cc:337] Failed RPC negotiation. Trace:
22:18:43.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.156410 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:55792 (local address 127.22.254.62:37215)
22:18:43.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.170471 (+ 14061us) server_negotiation.cc:206] Beginning negotiation
22:18:43.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.170491 (+ 20us) server_negotiation.cc:399] Waiting for connection header
22:18:43.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.170543 (+ 52us) server_negotiation.cc:407] Connection header received
22:18:43.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.170652 (+ 109us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:18:43.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.170676 (+ 24us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:18:43.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.170860 (+ 184us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:18:43.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.171024 (+ 164us) server_negotiation.cc:226] Negotiated authn=SASL
22:18:43.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.172590 (+ 1566us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:55792: BlockingRecv error: recv got EOF from 127.0.0.1:55792 (error 108)
22:18:43.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":13720}
22:18:43.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:43.175247 24149 negotiation.cc:337] Failed RPC negotiation. Trace:
22:18:43.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.166321 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:44486 (local address 127.22.254.61:36153)
22:18:43.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.166957 (+ 636us) server_negotiation.cc:206] Beginning negotiation
22:18:43.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.166987 (+ 30us) server_negotiation.cc:399] Waiting for connection header
22:18:43.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.167048 (+ 61us) server_negotiation.cc:407] Connection header received
22:18:43.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.167190 (+ 142us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:18:43.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.167223 (+ 33us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:18:43.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.167473 (+ 250us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:18:43.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.167691 (+ 218us) server_negotiation.cc:226] Negotiated authn=SASL
22:18:43.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.173951 (+ 6260us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:44486: BlockingRecv error: recv got EOF from 127.0.0.1:44486 (error 108)
22:18:43.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":132}
22:18:43.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:43.187400 24164 catalog_manager.cc:1259] Loaded cluster ID: 3feed9418f774d00b5c787fb3c6c3065
22:18:43.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:43.187649 24164 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 61f1a35769dd4952861645014e55e229: loading cluster ID for follower catalog manager: success
22:18:43.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:43.195250 24164 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 61f1a35769dd4952861645014e55e229: acquiring CA information for follower catalog manager: success
22:18:43.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:43.201678 24164 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 61f1a35769dd4952861645014e55e229: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
22:18:43.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:18:43 dist-test-slave-qzth krb5kdc[23555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547522, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.254.61@KRBTEST.COM for kudu/127.22.254.61@KRBTEST.COM
22:18:43.253 [DEBUG - pool-70-thread-1] (Connection.java:748) [peer master-127.22.254.62:37215(127.22.254.62:37215)] connecting to peer
22:18:43.254 [DEBUG - pool-70-thread-1] (Connection.java:748) [peer master-127.22.254.61:36153(127.22.254.61:36153)] connecting to peer
22:18:43.255 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.22.254.62:37215(127.22.254.62:37215)] Successfully connected to peer
22:18:43.255 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.22.254.61:36153(127.22.254.61:36153)] Successfully connected to peer
22:18:43.258 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.254.62:37215(127.22.254.62:37215)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:43.258 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.254.61:36153(127.22.254.61:36153)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:43.258 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.62:37215: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:43.258 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.61:36153: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:43.259 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.22.254.62:37215(127.22.254.62:37215)] handling channelInactive
22:18:43.259 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.22.254.61:36153(127.22.254.61:36153)] handling channelInactive
22:18:43.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:43.260811 24149 negotiation.cc:337] Failed RPC negotiation. Trace:
22:18:43.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.257106 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:44508 (local address 127.22.254.61:36153)
22:18:43.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.257559 (+ 453us) server_negotiation.cc:206] Beginning negotiation
22:18:43.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.257581 (+ 22us) server_negotiation.cc:399] Waiting for connection header
22:18:43.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.257620 (+ 39us) server_negotiation.cc:407] Connection header received
22:18:43.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.257737 (+ 117us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:18:43.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.257760 (+ 23us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:18:43.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.257972 (+ 212us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:18:43.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.259330 (+ 1358us) server_negotiation.cc:226] Negotiated authn=SASL
22:18:43.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.259943 (+ 613us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:44508: BlockingRecv error: recv got EOF from 127.0.0.1:44508 (error 108)
22:18:43.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":109}
22:18:43.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:43.266306 24165 negotiation.cc:337] Failed RPC negotiation. Trace:
22:18:43.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.256126 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:55808 (local address 127.22.254.62:37215)
22:18:43.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.256622 (+ 496us) server_negotiation.cc:206] Beginning negotiation
22:18:43.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.256646 (+ 24us) server_negotiation.cc:399] Waiting for connection header
22:18:43.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.256686 (+ 40us) server_negotiation.cc:407] Connection header received
22:18:43.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.256813 (+ 127us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:18:43.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.256836 (+ 23us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:18:43.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.257045 (+ 209us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:18:43.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.257209 (+ 164us) server_negotiation.cc:226] Negotiated authn=SASL
22:18:43.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.265359 (+ 8150us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:55808: BlockingRecv error: recv got EOF from 127.0.0.1:55808 (error 108)
22:18:43.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":118}
22:18:43.268 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153; will retry
22:18:43.414 [DEBUG - pool-70-thread-1] (Connection.java:748) [peer master-127.22.254.62:37215(127.22.254.62:37215)] connecting to peer
22:18:43.414 [DEBUG - pool-70-thread-1] (Connection.java:748) [peer master-127.22.254.61:36153(127.22.254.61:36153)] connecting to peer
22:18:43.415 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.22.254.62:37215(127.22.254.62:37215)] Successfully connected to peer
22:18:43.415 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.22.254.61:36153(127.22.254.61:36153)] Successfully connected to peer
22:18:43.418 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.254.62:37215(127.22.254.62:37215)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:43.418 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.254.61:36153(127.22.254.61:36153)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:43.418 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.62:37215: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:43.418 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.61:36153: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:43.418 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.22.254.62:37215(127.22.254.62:37215)] handling channelInactive
22:18:43.419 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.22.254.61:36153(127.22.254.61:36153)] handling channelInactive
22:18:43.419 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153; will retry
22:18:43.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:43.420545 24149 negotiation.cc:337] Failed RPC negotiation. Trace:
22:18:43.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.416143 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:44516 (local address 127.22.254.61:36153)
22:18:43.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.416738 (+ 595us) server_negotiation.cc:206] Beginning negotiation
22:18:43.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.416766 (+ 28us) server_negotiation.cc:399] Waiting for connection header
22:18:43.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.416811 (+ 45us) server_negotiation.cc:407] Connection header received
22:18:43.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.416934 (+ 123us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:18:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.416958 (+ 24us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:18:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.417173 (+ 215us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:18:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.417335 (+ 162us) server_negotiation.cc:226] Negotiated authn=SASL
22:18:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.419625 (+ 2290us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:44516: BlockingRecv error: recv got EOF from 127.0.0.1:44516 (error 108)
22:18:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":217}
22:18:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:43.420825 24165 negotiation.cc:337] Failed RPC negotiation. Trace:
22:18:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.416026 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:55820 (local address 127.22.254.62:37215)
22:18:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.416551 (+ 525us) server_negotiation.cc:206] Beginning negotiation
22:18:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.416574 (+ 23us) server_negotiation.cc:399] Waiting for connection header
22:18:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.416616 (+ 42us) server_negotiation.cc:407] Connection header received
22:18:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.416752 (+ 136us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:18:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.416776 (+ 24us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:18:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.417023 (+ 247us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:18:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.417207 (+ 184us) server_negotiation.cc:226] Negotiated authn=SASL
22:18:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.420141 (+ 2934us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:55820: BlockingRecv error: recv got EOF from 127.0.0.1:55820 (error 108)
22:18:43.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":126}
22:18:43.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:43.629297 24075 sys_catalog.cc:455] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "343dd293809c42459b96126818d74182" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "343dd293809c42459b96126818d74182" member_type: VOTER last_known_addr { host: "127.22.254.62" port: 37215 } } peers { permanent_uuid: "61f1a35769dd4952861645014e55e229" member_type: VOTER last_known_addr { host: "127.22.254.61" port: 36153 } } peers { permanent_uuid: "0cb22a9de93b4bfb93a644663cb469af" member_type: VOTER last_known_addr { host: "127.22.254.60" port: 44023 } } }
22:18:43.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:43.630026 24075 sys_catalog.cc:458] T 00000000000000000000000000000000 P 343dd293809c42459b96126818d74182 [sys.catalog]: This master's current role is: LEADER
22:18:43.633 [DEBUG - pool-70-thread-1] (Connection.java:748) [peer master-127.22.254.62:37215(127.22.254.62:37215)] connecting to peer
22:18:43.634 [DEBUG - pool-70-thread-1] (Connection.java:748) [peer master-127.22.254.61:36153(127.22.254.61:36153)] connecting to peer
22:18:43.634 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.22.254.62:37215(127.22.254.62:37215)] Successfully connected to peer
22:18:43.634 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.22.254.61:36153(127.22.254.61:36153)] Successfully connected to peer
22:18:43.637 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.254.62:37215(127.22.254.62:37215)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:43.637 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.62:37215: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:43.637 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.22.254.62:37215(127.22.254.62:37215)] handling channelInactive
22:18:43.638 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.254.61:36153(127.22.254.61:36153)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:43.638 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.61:36153: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:43.639 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153; will retry
22:18:43.639 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.22.254.61:36153(127.22.254.61:36153)] handling channelInactive
22:18:43.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:43.640317 24149 negotiation.cc:337] Failed RPC negotiation. Trace:
22:18:43.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.636312 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:44524 (local address 127.22.254.61:36153)
22:18:43.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.637372 (+ 1060us) server_negotiation.cc:206] Beginning negotiation
22:18:43.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.637397 (+ 25us) server_negotiation.cc:399] Waiting for connection header
22:18:43.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.637443 (+ 46us) server_negotiation.cc:407] Connection header received
22:18:43.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.637560 (+ 117us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:18:43.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.637586 (+ 26us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:18:43.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.637813 (+ 227us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:18:43.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.638010 (+ 197us) server_negotiation.cc:226] Negotiated authn=SASL
22:18:43.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.639694 (+ 1684us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:44524: BlockingRecv error: recv got EOF from 127.0.0.1:44524 (error 108)
22:18:43.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":668}
22:18:43.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:43.640322 24165 negotiation.cc:337] Failed RPC negotiation. Trace:
22:18:43.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.635523 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:55822 (local address 127.22.254.62:37215)
22:18:43.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.636048 (+ 525us) server_negotiation.cc:206] Beginning negotiation
22:18:43.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.636072 (+ 24us) server_negotiation.cc:399] Waiting for connection header
22:18:43.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.636116 (+ 44us) server_negotiation.cc:407] Connection header received
22:18:43.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.636256 (+ 140us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:18:43.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.636282 (+ 26us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:18:43.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.636506 (+ 224us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:18:43.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.636669 (+ 163us) server_negotiation.cc:226] Negotiated authn=SASL
22:18:43.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:43.639363 (+ 2694us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:55822: BlockingRecv error: recv got EOF from 127.0.0.1:55822 (error 108)
22:18:43.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":126}
22:18:44.074 [DEBUG - pool-70-thread-1] (Connection.java:748) [peer master-127.22.254.62:37215(127.22.254.62:37215)] connecting to peer
22:18:44.074 [DEBUG - pool-70-thread-1] (Connection.java:748) [peer master-127.22.254.61:36153(127.22.254.61:36153)] connecting to peer
22:18:44.075 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.22.254.61:36153(127.22.254.61:36153)] Successfully connected to peer
22:18:44.076 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.22.254.62:37215(127.22.254.62:37215)] Successfully connected to peer
22:18:44.078 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.254.62:37215(127.22.254.62:37215)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:44.078 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.62:37215: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:44.078 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.254.61:36153(127.22.254.61:36153)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:44.079 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.61:36153: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:44.079 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.22.254.62:37215(127.22.254.62:37215)] handling channelInactive
22:18:44.079 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.22.254.61:36153(127.22.254.61:36153)] handling channelInactive
22:18:44.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:44.080873 24149 negotiation.cc:337] Failed RPC negotiation. Trace:
22:18:44.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:44.077025 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:44526 (local address 127.22.254.61:36153)
22:18:44.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:44.077655 (+ 630us) server_negotiation.cc:206] Beginning negotiation
22:18:44.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:44.077691 (+ 36us) server_negotiation.cc:399] Waiting for connection header
22:18:44.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:44.077763 (+ 72us) server_negotiation.cc:407] Connection header received
22:18:44.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:44.078072 (+ 309us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:18:44.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:44.078103 (+ 31us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:18:44.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:44.078375 (+ 272us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:18:44.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:44.078610 (+ 235us) server_negotiation.cc:226] Negotiated authn=SASL
22:18:44.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:44.080036 (+ 1426us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:44526: BlockingRecv error: recv got EOF from 127.0.0.1:44526 (error 108)
22:18:44.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":146}
22:18:44.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:44.081328 24165 negotiation.cc:337] Failed RPC negotiation. Trace:
22:18:44.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:44.076875 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:55838 (local address 127.22.254.62:37215)
22:18:44.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:44.077419 (+ 544us) server_negotiation.cc:206] Beginning negotiation
22:18:44.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:44.077446 (+ 27us) server_negotiation.cc:399] Waiting for connection header
22:18:44.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:44.077497 (+ 51us) server_negotiation.cc:407] Connection header received
22:18:44.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:44.077632 (+ 135us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:18:44.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:44.077658 (+ 26us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:18:44.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:44.077913 (+ 255us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:18:44.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:44.078094 (+ 181us) server_negotiation.cc:226] Negotiated authn=SASL
22:18:44.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:44.080295 (+ 2201us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:55838: BlockingRecv error: recv got EOF from 127.0.0.1:55838 (error 108)
22:18:44.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":127}
22:18:44.082 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153; will retry
22:18:44.793 [DEBUG - pool-70-thread-1] (Connection.java:748) [peer master-127.22.254.62:37215(127.22.254.62:37215)] connecting to peer
22:18:44.794 [DEBUG - pool-70-thread-1] (Connection.java:748) [peer master-127.22.254.61:36153(127.22.254.61:36153)] connecting to peer
22:18:44.795 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.22.254.62:37215(127.22.254.62:37215)] Successfully connected to peer
22:18:44.798 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.254.62:37215(127.22.254.62:37215)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:44.799 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.62:37215: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:44.799 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.22.254.61:36153(127.22.254.61:36153)] Successfully connected to peer
22:18:44.799 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.22.254.62:37215(127.22.254.62:37215)] handling channelInactive
22:18:44.800 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.254.61:36153(127.22.254.61:36153)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:44.800 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.61:36153: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:44.800 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153; will retry
22:18:44.801 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.22.254.61:36153(127.22.254.61:36153)] handling channelInactive
22:18:44.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:44.801717 24175 negotiation.cc:337] Failed RPC negotiation. Trace:
22:18:44.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:44.796409 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:55842 (local address 127.22.254.62:37215)
22:18:44.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:44.797581 (+ 1172us) server_negotiation.cc:206] Beginning negotiation
22:18:44.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:44.797603 (+ 22us) server_negotiation.cc:399] Waiting for connection header
22:18:44.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:44.797650 (+ 47us) server_negotiation.cc:407] Connection header received
22:18:44.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:44.797802 (+ 152us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:18:44.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:44.797864 (+ 62us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:18:44.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:44.798162 (+ 298us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:18:44.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:44.799932 (+ 1770us) server_negotiation.cc:226] Negotiated authn=SASL
22:18:44.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:44.800574 (+ 642us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:55842: BlockingRecv error: recv got EOF from 127.0.0.1:55842 (error 108)
22:18:44.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":778,"thread_start_us":499,"threads_started":1}
22:18:44.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:44.802835 24176 negotiation.cc:337] Failed RPC negotiation. Trace:
22:18:44.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:44.797457 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:44542 (local address 127.22.254.61:36153)
22:18:44.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:44.799352 (+ 1895us) server_negotiation.cc:206] Beginning negotiation
22:18:44.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:44.799379 (+ 27us) server_negotiation.cc:399] Waiting for connection header
22:18:44.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:44.799425 (+ 46us) server_negotiation.cc:407] Connection header received
22:18:44.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:44.799569 (+ 144us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:18:44.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:44.799645 (+ 76us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:18:44.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:44.799871 (+ 226us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:18:44.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:44.800074 (+ 203us) server_negotiation.cc:226] Negotiated authn=SASL
22:18:44.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:44.801534 (+ 1460us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:44542: BlockingRecv error: recv got EOF from 127.0.0.1:44542 (error 108)
22:18:44.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":1415,"thread_start_us":1191,"threads_started":1}
22:18:46.053 [DEBUG - pool-70-thread-1] (Connection.java:748) [peer master-127.22.254.62:37215(127.22.254.62:37215)] connecting to peer
22:18:46.054 [DEBUG - pool-70-thread-1] (Connection.java:748) [peer master-127.22.254.61:36153(127.22.254.61:36153)] connecting to peer
22:18:46.055 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.22.254.62:37215(127.22.254.62:37215)] Successfully connected to peer
22:18:46.055 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.22.254.61:36153(127.22.254.61:36153)] Successfully connected to peer
22:18:46.058 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.254.62:37215(127.22.254.62:37215)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:46.058 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.62:37215: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:46.058 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.22.254.62:37215(127.22.254.62:37215)] handling channelInactive
22:18:46.059 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.254.61:36153(127.22.254.61:36153)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:46.059 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.61:36153: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:46.059 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153; will retry
22:18:46.059 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.22.254.61:36153(127.22.254.61:36153)] handling channelInactive
22:18:46.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:46.059994 24179 negotiation.cc:337] Failed RPC negotiation. Trace:
22:18:46.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:46.055257 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:55848 (local address 127.22.254.62:37215)
22:18:46.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:46.057032 (+ 1775us) server_negotiation.cc:206] Beginning negotiation
22:18:46.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:46.057060 (+ 28us) server_negotiation.cc:399] Waiting for connection header
22:18:46.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:46.057106 (+ 46us) server_negotiation.cc:407] Connection header received
22:18:46.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:46.057312 (+ 206us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:18:46.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:46.057336 (+ 24us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:18:46.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:46.057565 (+ 229us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:18:46.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:46.057736 (+ 171us) server_negotiation.cc:226] Negotiated authn=SASL
22:18:46.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:46.059084 (+ 1348us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:55848: BlockingRecv error: recv got EOF from 127.0.0.1:55848 (error 108)
22:18:46.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":1362,"thread_start_us":1506,"threads_started":1}
22:18:46.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:46.060881 24180 negotiation.cc:337] Failed RPC negotiation. Trace:
22:18:46.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:46.055668 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:44550 (local address 127.22.254.61:36153)
22:18:46.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:46.058299 (+ 2631us) server_negotiation.cc:206] Beginning negotiation
22:18:46.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:46.058313 (+ 14us) server_negotiation.cc:399] Waiting for connection header
22:18:46.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:46.058347 (+ 34us) server_negotiation.cc:407] Connection header received
22:18:46.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:46.058443 (+ 96us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:18:46.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:46.058459 (+ 16us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:18:46.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:46.058611 (+ 152us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:18:46.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:46.058786 (+ 175us) server_negotiation.cc:226] Negotiated authn=SASL
22:18:46.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:46.059901 (+ 1115us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:44550: BlockingRecv error: recv got EOF from 127.0.0.1:44550 (error 108)
22:18:46.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":2271,"thread_start_us":2004,"threads_started":1}
22:18:48.293 [DEBUG - pool-70-thread-1] (Connection.java:748) [peer master-127.22.254.62:37215(127.22.254.62:37215)] connecting to peer
22:18:48.294 [DEBUG - pool-70-thread-1] (Connection.java:748) [peer master-127.22.254.61:36153(127.22.254.61:36153)] connecting to peer
22:18:48.294 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.22.254.62:37215(127.22.254.62:37215)] Successfully connected to peer
22:18:48.294 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.22.254.61:36153(127.22.254.61:36153)] Successfully connected to peer
22:18:48.298 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.254.61:36153(127.22.254.61:36153)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:48.298 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.61:36153: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:48.298 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.254.62:37215(127.22.254.62:37215)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:48.298 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.22.254.61:36153(127.22.254.61:36153)] handling channelInactive
22:18:48.298 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.62:37215: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:48.299 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153; will retry
22:18:48.299 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.22.254.62:37215(127.22.254.62:37215)] handling channelInactive
22:18:48.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:48.301043 24190 negotiation.cc:337] Failed RPC negotiation. Trace:
22:18:48.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:48.295900 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:44560 (local address 127.22.254.61:36153)
22:18:48.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:48.297323 (+ 1423us) server_negotiation.cc:206] Beginning negotiation
22:18:48.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:48.297348 (+ 25us) server_negotiation.cc:399] Waiting for connection header
22:18:48.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:48.297389 (+ 41us) server_negotiation.cc:407] Connection header received
22:18:48.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:48.297487 (+ 98us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:18:48.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:48.297503 (+ 16us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:18:48.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:48.297645 (+ 142us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:18:48.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:48.297769 (+ 124us) server_negotiation.cc:226] Negotiated authn=SASL
22:18:48.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:48.299919 (+ 2150us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:44560: BlockingRecv error: recv got EOF from 127.0.0.1:44560 (error 108)
22:18:48.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":795,"thread_start_us":524,"threads_started":1}
22:18:48.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:48.301506 24191 negotiation.cc:337] Failed RPC negotiation. Trace:
22:18:48.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:48.295989 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:55858 (local address 127.22.254.62:37215)
22:18:48.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:48.297658 (+ 1669us) server_negotiation.cc:206] Beginning negotiation
22:18:48.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:48.297676 (+ 18us) server_negotiation.cc:399] Waiting for connection header
22:18:48.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:48.297717 (+ 41us) server_negotiation.cc:407] Connection header received
22:18:48.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:48.297950 (+ 233us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:18:48.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:48.297973 (+ 23us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:18:48.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:48.298205 (+ 232us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:18:48.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:48.298406 (+ 201us) server_negotiation.cc:226] Negotiated authn=SASL
22:18:48.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:48.300339 (+ 1933us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:55858: BlockingRecv error: recv got EOF from 127.0.0.1:55858 (error 108)
22:18:48.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":1316,"thread_start_us":952,"threads_started":1}
22:18:48.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:18:48 dist-test-slave-qzth krb5kdc[23555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547528, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.254.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
22:18:48.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:48.738333 23892 init.cc:295] Successfully reacquired a new kerberos TGT
22:18:48.873 [DEBUG - pool-70-thread-1] (Connection.java:748) [peer master-127.22.254.62:37215(127.22.254.62:37215)] connecting to peer
22:18:48.874 [DEBUG - pool-70-thread-1] (Connection.java:748) [peer master-127.22.254.61:36153(127.22.254.61:36153)] connecting to peer
22:18:48.875 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.22.254.62:37215(127.22.254.62:37215)] Successfully connected to peer
22:18:48.875 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.22.254.61:36153(127.22.254.61:36153)] Successfully connected to peer
22:18:48.879 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.254.61:36153(127.22.254.61:36153)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:48.879 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.61:36153: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:48.879 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.22.254.61:36153(127.22.254.61:36153)] handling channelInactive
22:18:48.881 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.254.62:37215(127.22.254.62:37215)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:48.881 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.62:37215: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:48.881 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153; will retry
22:18:48.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:48.881846 24193 negotiation.cc:337] Failed RPC negotiation. Trace:
22:18:48.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:48.876059 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:44562 (local address 127.22.254.61:36153)
22:18:48.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:48.877811 (+ 1752us) server_negotiation.cc:206] Beginning negotiation
22:18:48.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:48.877851 (+ 40us) server_negotiation.cc:399] Waiting for connection header
22:18:48.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:48.877984 (+ 133us) server_negotiation.cc:407] Connection header received
22:18:48.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:48.878127 (+ 143us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:18:48.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:48.878152 (+ 25us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:18:48.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:48.878441 (+ 289us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:18:48.882 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.22.254.62:37215(127.22.254.62:37215)] handling channelInactive
22:18:48.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:48.880121 (+ 1680us) server_negotiation.cc:226] Negotiated authn=SASL
22:18:48.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:48.880786 (+ 665us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:44562: BlockingRecv error: recv got EOF from 127.0.0.1:44562 (error 108)
22:18:48.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":1295,"thread_start_us":1003,"threads_started":1}
22:18:48.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:48.884100 24194 negotiation.cc:337] Failed RPC negotiation. Trace:
22:18:48.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:48.877635 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:55870 (local address 127.22.254.62:37215)
22:18:48.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:48.880406 (+ 2771us) server_negotiation.cc:206] Beginning negotiation
22:18:48.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:48.880425 (+ 19us) server_negotiation.cc:399] Waiting for connection header
22:18:48.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:48.880469 (+ 44us) server_negotiation.cc:407] Connection header received
22:18:48.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:48.880596 (+ 127us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:18:48.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:48.880623 (+ 27us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:18:48.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:48.880849 (+ 226us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:18:48.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:48.882514 (+ 1665us) server_negotiation.cc:226] Negotiated authn=SASL
22:18:48.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:48.883116 (+ 602us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:55870: BlockingRecv error: recv got EOF from 127.0.0.1:55870 (error 108)
22:18:48.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":2383,"thread_start_us":2115,"threads_started":1}
22:18:50.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:18:50 dist-test-slave-qzth krb5kdc[23555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547530, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.254.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
22:18:50.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:50.704923 23984 init.cc:295] Successfully reacquired a new kerberos TGT
22:18:52.073 [DEBUG - pool-70-thread-1] (Connection.java:748) [peer master-127.22.254.62:37215(127.22.254.62:37215)] connecting to peer
22:18:52.074 [DEBUG - pool-70-thread-1] (Connection.java:748) [peer master-127.22.254.61:36153(127.22.254.61:36153)] connecting to peer
22:18:52.074 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.22.254.61:36153(127.22.254.61:36153)] Successfully connected to peer
22:18:52.074 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.22.254.62:37215(127.22.254.62:37215)] Successfully connected to peer
22:18:52.078 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.254.62:37215(127.22.254.62:37215)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:52.078 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.62:37215: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:52.078 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.22.254.62:37215(127.22.254.62:37215)] handling channelInactive
22:18:52.079 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.254.61:36153(127.22.254.61:36153)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:52.079 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.61:36153: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:52.079 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153; will retry
22:18:52.080 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.22.254.61:36153(127.22.254.61:36153)] handling channelInactive
22:18:52.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:52.079906 24199 negotiation.cc:337] Failed RPC negotiation. Trace:
22:18:52.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:52.075184 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:35752 (local address 127.22.254.62:37215)
22:18:52.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:52.076763 (+ 1579us) server_negotiation.cc:206] Beginning negotiation
22:18:52.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:52.076794 (+ 31us) server_negotiation.cc:399] Waiting for connection header
22:18:52.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:52.076862 (+ 68us) server_negotiation.cc:407] Connection header received
22:18:52.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:52.077046 (+ 184us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:18:52.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:52.077084 (+ 38us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:18:52.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:52.077364 (+ 280us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:18:52.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:52.077643 (+ 279us) server_negotiation.cc:226] Negotiated authn=SASL
22:18:52.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:52.078865 (+ 1222us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:35752: BlockingRecv error: recv got EOF from 127.0.0.1:35752 (error 108)
22:18:52.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":879,"thread_start_us":537,"threads_started":1}
22:18:52.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:52.081410 24200 negotiation.cc:337] Failed RPC negotiation. Trace:
22:18:52.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:52.075677 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:53196 (local address 127.22.254.61:36153)
22:18:52.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:52.078076 (+ 2399us) server_negotiation.cc:206] Beginning negotiation
22:18:52.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:52.078114 (+ 38us) server_negotiation.cc:399] Waiting for connection header
22:18:52.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:52.078175 (+ 61us) server_negotiation.cc:407] Connection header received
22:18:52.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:52.078360 (+ 185us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:18:52.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:52.078402 (+ 42us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:18:52.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:52.078650 (+ 248us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:18:52.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:52.078850 (+ 200us) server_negotiation.cc:226] Negotiated authn=SASL
22:18:52.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:52.080213 (+ 1363us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:53196: BlockingRecv error: recv got EOF from 127.0.0.1:53196 (error 108)
22:18:52.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":1872,"thread_start_us":1519,"threads_started":1}
22:18:52.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:18:52 dist-test-slave-qzth krb5kdc[23555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547532, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.254.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
22:18:52.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:52.706797 24083 init.cc:295] Successfully reacquired a new kerberos TGT
22:18:53.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:53.873965 24036 debug-util.cc:398] Leaking SignalData structure 0x7b080008bf00 after lost signal to thread 23967
22:18:53.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:53.874554 24036 debug-util.cc:398] Leaking SignalData structure 0x7b08000b70e0 after lost signal to thread 24039
22:18:55.894 [DEBUG - pool-70-thread-1] (Connection.java:748) [peer master-127.22.254.62:37215(127.22.254.62:37215)] connecting to peer
22:18:55.894 [DEBUG - pool-70-thread-1] (Connection.java:748) [peer master-127.22.254.61:36153(127.22.254.61:36153)] connecting to peer
22:18:55.895 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.22.254.61:36153(127.22.254.61:36153)] Successfully connected to peer
22:18:55.895 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.22.254.62:37215(127.22.254.62:37215)] Successfully connected to peer
22:18:55.911 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.254.61:36153(127.22.254.61:36153)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:55.911 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.61:36153: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:55.911 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.22.254.61:36153(127.22.254.61:36153)] handling channelInactive
22:18:55.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:55.914047 24206 negotiation.cc:337] Failed RPC negotiation. Trace:
22:18:55.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:55.896626 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:53202 (local address 127.22.254.61:36153)
22:18:55.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:55.902317 (+ 5691us) server_negotiation.cc:206] Beginning negotiation
22:18:55.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:55.902343 (+ 26us) server_negotiation.cc:399] Waiting for connection header
22:18:55.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:55.902394 (+ 51us) server_negotiation.cc:407] Connection header received
22:18:55.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:55.902588 (+ 194us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:18:55.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:55.902613 (+ 25us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:18:55.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:55.902822 (+ 209us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:18:55.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:55.912112 (+ 9290us) server_negotiation.cc:226] Negotiated authn=SASL
22:18:55.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:55.912833 (+ 721us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:53202: BlockingRecv error: recv got EOF from 127.0.0.1:53202 (error 108)
22:18:55.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":5187,"thread_start_us":5416,"threads_started":1}
22:18:55.918 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.254.62:37215(127.22.254.62:37215)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:55.918 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.62:37215: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:55.919 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153; will retry
22:18:55.919 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.22.254.62:37215(127.22.254.62:37215)] handling channelInactive
22:18:55.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:55.920794 24207 negotiation.cc:337] Failed RPC negotiation. Trace:
22:18:55.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:55.896781 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:35756 (local address 127.22.254.62:37215)
22:18:55.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:55.917629 (+ 20848us) server_negotiation.cc:206] Beginning negotiation
22:18:55.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:55.917656 (+ 27us) server_negotiation.cc:399] Waiting for connection header
22:18:55.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:55.917703 (+ 47us) server_negotiation.cc:407] Connection header received
22:18:55.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:55.917858 (+ 155us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:18:55.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:55.917884 (+ 26us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:18:55.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:55.918084 (+ 200us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:18:55.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:55.918266 (+ 182us) server_negotiation.cc:226] Negotiated authn=SASL
22:18:55.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:55.919685 (+ 1419us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:35756: BlockingRecv error: recv got EOF from 127.0.0.1:35756 (error 108)
22:18:55.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":20403,"thread_start_us":20995,"threads_started":1}
22:18:58.613 [DEBUG - pool-70-thread-1] (Connection.java:748) [peer master-127.22.254.62:37215(127.22.254.62:37215)] connecting to peer
22:18:58.614 [DEBUG - pool-70-thread-1] (Connection.java:748) [peer master-127.22.254.61:36153(127.22.254.61:36153)] connecting to peer
22:18:58.615 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.22.254.62:37215(127.22.254.62:37215)] Successfully connected to peer
22:18:58.615 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.22.254.61:36153(127.22.254.61:36153)] Successfully connected to peer
22:18:58.618 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.254.62:37215(127.22.254.62:37215)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:58.619 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.254.61:36153(127.22.254.61:36153)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:58.619 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.61:36153: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:58.619 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.22.254.61:36153(127.22.254.61:36153)] handling channelInactive
22:18:58.620 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.62:37215: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:58.620 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153; will retry
22:18:58.620 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.22.254.62:37215(127.22.254.62:37215)] handling channelInactive
22:18:58.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:58.621233 24212 negotiation.cc:337] Failed RPC negotiation. Trace:
22:18:58.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:58.616090 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:53212 (local address 127.22.254.61:36153)
22:18:58.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:58.617862 (+ 1772us) server_negotiation.cc:206] Beginning negotiation
22:18:58.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:58.617885 (+ 23us) server_negotiation.cc:399] Waiting for connection header
22:18:58.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:58.617937 (+ 52us) server_negotiation.cc:407] Connection header received
22:18:58.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:58.618066 (+ 129us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:18:58.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:58.618091 (+ 25us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:18:58.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:58.618329 (+ 238us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:18:58.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:58.618621 (+ 292us) server_negotiation.cc:226] Negotiated authn=SASL
22:18:58.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:58.620190 (+ 1569us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:53212: BlockingRecv error: recv got EOF from 127.0.0.1:53212 (error 108)
22:18:58.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":1337,"thread_start_us":1077,"threads_started":1}
22:18:58.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:58.622103 24211 negotiation.cc:337] Failed RPC negotiation. Trace:
22:18:58.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:58.616079 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:35770 (local address 127.22.254.62:37215)
22:18:58.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:58.617551 (+ 1472us) server_negotiation.cc:206] Beginning negotiation
22:18:58.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:58.617579 (+ 28us) server_negotiation.cc:399] Waiting for connection header
22:18:58.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:58.617644 (+ 65us) server_negotiation.cc:407] Connection header received
22:18:58.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:58.617865 (+ 221us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:18:58.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:58.617890 (+ 25us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:18:58.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:58.618188 (+ 298us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:18:58.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:58.619043 (+ 855us) server_negotiation.cc:226] Negotiated authn=SASL
22:18:58.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:58.621073 (+ 2030us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:35770: BlockingRecv error: recv got EOF from 127.0.0.1:35770 (error 108)
22:18:58.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":907,"thread_start_us":605,"threads_started":1}
22:18:58.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:18:58 dist-test-slave-qzth krb5kdc[23555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547538, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.254.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
22:18:58.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:18:58.740582 23892 init.cc:295] Successfully reacquired a new kerberos TGT
22:18:59.833 [DEBUG - pool-70-thread-1] (Connection.java:748) [peer master-127.22.254.62:37215(127.22.254.62:37215)] connecting to peer
22:18:59.833 [DEBUG - pool-70-thread-1] (Connection.java:748) [peer master-127.22.254.61:36153(127.22.254.61:36153)] connecting to peer
22:18:59.834 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.22.254.61:36153(127.22.254.61:36153)] Successfully connected to peer
22:18:59.834 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.22.254.62:37215(127.22.254.62:37215)] Successfully connected to peer
22:18:59.840 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.254.61:36153(127.22.254.61:36153)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:59.840 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.61:36153: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:59.840 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.254.62:37215(127.22.254.62:37215)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:59.840 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.22.254.61:36153(127.22.254.61:36153)] handling channelInactive
22:18:59.840 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.62:37215: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:18:59.840 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153; will retry
22:18:59.841 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.22.254.62:37215(127.22.254.62:37215)] handling channelInactive
22:18:59.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:59.841924 24218 negotiation.cc:337] Failed RPC negotiation. Trace:
22:18:59.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:59.834856 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:53228 (local address 127.22.254.61:36153)
22:18:59.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:59.839041 (+ 4185us) server_negotiation.cc:206] Beginning negotiation
22:18:59.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:59.839062 (+ 21us) server_negotiation.cc:399] Waiting for connection header
22:18:59.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:59.839105 (+ 43us) server_negotiation.cc:407] Connection header received
22:18:59.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:59.839243 (+ 138us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:18:59.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:59.839266 (+ 23us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:18:59.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:59.839535 (+ 269us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:18:59.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:59.839672 (+ 137us) server_negotiation.cc:226] Negotiated authn=SASL
22:18:59.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:59.840842 (+ 1170us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:53228: BlockingRecv error: recv got EOF from 127.0.0.1:53228 (error 108)
22:18:59.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":3702,"thread_start_us":3360,"threads_started":1}
22:18:59.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:18:59.842571 24219 negotiation.cc:337] Failed RPC negotiation. Trace:
22:18:59.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:59.835155 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:35776 (local address 127.22.254.62:37215)
22:18:59.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:59.839399 (+ 4244us) server_negotiation.cc:206] Beginning negotiation
22:18:59.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:59.839414 (+ 15us) server_negotiation.cc:399] Waiting for connection header
22:18:59.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:59.839472 (+ 58us) server_negotiation.cc:407] Connection header received
22:18:59.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:59.839590 (+ 118us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:18:59.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:59.839606 (+ 16us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:18:59.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:59.839832 (+ 226us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:18:59.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:59.840019 (+ 187us) server_negotiation.cc:226] Negotiated authn=SASL
22:18:59.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:18:59.841585 (+ 1566us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:35776: BlockingRecv error: recv got EOF from 127.0.0.1:35776 (error 108)
22:18:59.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":3806,"thread_start_us":3494,"threads_started":1}
22:19:00.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:19:00 dist-test-slave-qzth krb5kdc[23555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547540, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.254.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
22:19:00.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:19:00.707188 23984 init.cc:295] Successfully reacquired a new kerberos TGT
22:19:02.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:19:02 dist-test-slave-qzth krb5kdc[23555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547542, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.254.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
22:19:02.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:19:02.709297 24083 init.cc:295] Successfully reacquired a new kerberos TGT
22:19:03.693 [DEBUG - pool-70-thread-1] (Connection.java:748) [peer master-127.22.254.62:37215(127.22.254.62:37215)] connecting to peer
22:19:03.694 [DEBUG - pool-70-thread-1] (Connection.java:748) [peer master-127.22.254.61:36153(127.22.254.61:36153)] connecting to peer
22:19:03.695 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.22.254.61:36153(127.22.254.61:36153)] Successfully connected to peer
22:19:03.695 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.22.254.62:37215(127.22.254.62:37215)] Successfully connected to peer
22:19:03.699 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.254.61:36153(127.22.254.61:36153)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:19:03.699 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.61:36153: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:19:03.699 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.22.254.61:36153(127.22.254.61:36153)] handling channelInactive
22:19:03.699 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.254.62:37215(127.22.254.62:37215)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:19:03.700 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.62:37215: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:19:03.700 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153; will retry
22:19:03.700 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.22.254.62:37215(127.22.254.62:37215)] handling channelInactive
22:19:03.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:19:03.702178 24225 negotiation.cc:337] Failed RPC negotiation. Trace:
22:19:03.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:03.695989 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:41364 (local address 127.22.254.62:37215)
22:19:03.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:03.698497 (+ 2508us) server_negotiation.cc:206] Beginning negotiation
22:19:03.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:03.698520 (+ 23us) server_negotiation.cc:399] Waiting for connection header
22:19:03.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:03.698562 (+ 42us) server_negotiation.cc:407] Connection header received
22:19:03.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:03.698688 (+ 126us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:19:03.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:03.698713 (+ 25us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:19:03.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:03.698929 (+ 216us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:19:03.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:03.699525 (+ 596us) server_negotiation.cc:226] Negotiated authn=SASL
22:19:03.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:03.700998 (+ 1473us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:41364: BlockingRecv error: recv got EOF from 127.0.0.1:41364 (error 108)
22:19:03.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":2075,"thread_start_us":1811,"threads_started":1}
22:19:03.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:19:03.702893 24226 negotiation.cc:337] Failed RPC negotiation. Trace:
22:19:03.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:03.696261 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:39994 (local address 127.22.254.61:36153)
22:19:03.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:03.698156 (+ 1895us) server_negotiation.cc:206] Beginning negotiation
22:19:03.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:03.698187 (+ 31us) server_negotiation.cc:399] Waiting for connection header
22:19:03.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:03.698251 (+ 64us) server_negotiation.cc:407] Connection header received
22:19:03.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:03.698440 (+ 189us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:19:03.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:03.698466 (+ 26us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:19:03.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:03.698713 (+ 247us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:19:03.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:03.700783 (+ 2070us) server_negotiation.cc:226] Negotiated authn=SASL
22:19:03.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:03.701613 (+ 830us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:39994: BlockingRecv error: recv got EOF from 127.0.0.1:39994 (error 108)
22:19:03.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":1366,"thread_start_us":1046,"threads_started":1}
22:19:05.253 [DEBUG - pool-70-thread-1] (Connection.java:748) [peer master-127.22.254.62:37215(127.22.254.62:37215)] connecting to peer
22:19:05.254 [DEBUG - pool-70-thread-1] (Connection.java:748) [peer master-127.22.254.61:36153(127.22.254.61:36153)] connecting to peer
22:19:05.254 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.22.254.62:37215(127.22.254.62:37215)] Successfully connected to peer
22:19:05.254 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.22.254.61:36153(127.22.254.61:36153)] Successfully connected to peer
22:19:05.257 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.254.61:36153(127.22.254.61:36153)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:19:05.258 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.61:36153: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:19:05.258 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.254.62:37215(127.22.254.62:37215)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:19:05.258 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.62:37215: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:19:05.258 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.22.254.61:36153(127.22.254.61:36153)] handling channelInactive
22:19:05.258 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153; will retry
22:19:05.259 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.22.254.62:37215(127.22.254.62:37215)] handling channelInactive
22:19:05.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:19:05.259572 24229 negotiation.cc:337] Failed RPC negotiation. Trace:
22:19:05.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:05.255566 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:40002 (local address 127.22.254.61:36153)
22:19:05.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:05.256839 (+ 1273us) server_negotiation.cc:206] Beginning negotiation
22:19:05.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:05.256861 (+ 22us) server_negotiation.cc:399] Waiting for connection header
22:19:05.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:05.256907 (+ 46us) server_negotiation.cc:407] Connection header received
22:19:05.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:05.257047 (+ 140us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:19:05.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:05.257075 (+ 28us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:19:05.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:05.257294 (+ 219us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:19:05.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:05.257478 (+ 184us) server_negotiation.cc:226] Negotiated authn=SASL
22:19:05.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:05.258643 (+ 1165us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:40002: BlockingRecv error: recv got EOF from 127.0.0.1:40002 (error 108)
22:19:05.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":843,"thread_start_us":585,"threads_started":1}
22:19:05.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:19:05.260058 24230 negotiation.cc:337] Failed RPC negotiation. Trace:
22:19:05.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:05.255651 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:41378 (local address 127.22.254.62:37215)
22:19:05.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:05.257267 (+ 1616us) server_negotiation.cc:206] Beginning negotiation
22:19:05.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:05.257289 (+ 22us) server_negotiation.cc:399] Waiting for connection header
22:19:05.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:05.257333 (+ 44us) server_negotiation.cc:407] Connection header received
22:19:05.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:05.257470 (+ 137us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:19:05.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:05.257495 (+ 25us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:19:05.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:05.257715 (+ 220us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:19:05.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:05.257942 (+ 227us) server_negotiation.cc:226] Negotiated authn=SASL
22:19:05.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:05.259198 (+ 1256us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:41378: BlockingRecv error: recv got EOF from 127.0.0.1:41378 (error 108)
22:19:05.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":1214,"thread_start_us":945,"threads_started":1}
22:19:08.273 [DEBUG - pool-70-thread-1] (Connection.java:748) [peer master-127.22.254.62:37215(127.22.254.62:37215)] connecting to peer
22:19:08.274 [DEBUG - pool-70-thread-1] (Connection.java:748) [peer master-127.22.254.61:36153(127.22.254.61:36153)] connecting to peer
22:19:08.274 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.22.254.62:37215(127.22.254.62:37215)] Successfully connected to peer
22:19:08.274 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.22.254.61:36153(127.22.254.61:36153)] Successfully connected to peer
22:19:08.280 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.254.62:37215(127.22.254.62:37215)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:19:08.281 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.62:37215: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:19:08.281 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.254.61:36153(127.22.254.61:36153)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:19:08.281 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.61:36153: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:19:08.281 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.22.254.62:37215(127.22.254.62:37215)] handling channelInactive
22:19:08.281 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153; will retry
22:19:08.281 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.22.254.61:36153(127.22.254.61:36153)] handling channelInactive
22:19:08.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:19:08.283025 24237 negotiation.cc:337] Failed RPC negotiation. Trace:
22:19:08.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:08.275978 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:41394 (local address 127.22.254.62:37215)
22:19:08.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:08.279565 (+ 3587us) server_negotiation.cc:206] Beginning negotiation
22:19:08.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:08.279591 (+ 26us) server_negotiation.cc:399] Waiting for connection header
22:19:08.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:08.279649 (+ 58us) server_negotiation.cc:407] Connection header received
22:19:08.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:08.279830 (+ 181us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:19:08.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:08.279853 (+ 23us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:19:08.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:08.280108 (+ 255us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:19:08.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:08.280295 (+ 187us) server_negotiation.cc:226] Negotiated authn=SASL
22:19:08.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:08.281732 (+ 1437us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:41394: BlockingRecv error: recv got EOF from 127.0.0.1:41394 (error 108)
22:19:08.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":3101,"thread_start_us":2786,"threads_started":1}
22:19:08.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:19:08.283250 24236 negotiation.cc:337] Failed RPC negotiation. Trace:
22:19:08.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:08.275951 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:40018 (local address 127.22.254.61:36153)
22:19:08.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:08.280025 (+ 4074us) server_negotiation.cc:206] Beginning negotiation
22:19:08.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:08.280040 (+ 15us) server_negotiation.cc:399] Waiting for connection header
22:19:08.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:08.280074 (+ 34us) server_negotiation.cc:407] Connection header received
22:19:08.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:08.280175 (+ 101us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:19:08.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:08.280190 (+ 15us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:19:08.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:08.280409 (+ 219us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:19:08.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:08.280748 (+ 339us) server_negotiation.cc:226] Negotiated authn=SASL
22:19:08.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:08.282131 (+ 1383us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:40018: BlockingRecv error: recv got EOF from 127.0.0.1:40018 (error 108)
22:19:08.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":3658,"thread_start_us":3376,"threads_started":1}
22:19:08.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:19:08 dist-test-slave-qzth krb5kdc[23555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547548, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.254.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
22:19:08.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:19:08.742821 23892 init.cc:295] Successfully reacquired a new kerberos TGT
22:19:10.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:19:10 dist-test-slave-qzth krb5kdc[23555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547550, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.254.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
22:19:10.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:19:10.709522 23984 init.cc:295] Successfully reacquired a new kerberos TGT
22:19:10.913 [DEBUG - pool-70-thread-1] (Connection.java:748) [peer master-127.22.254.62:37215(127.22.254.62:37215)] connecting to peer
22:19:10.914 [DEBUG - pool-70-thread-1] (Connection.java:748) [peer master-127.22.254.61:36153(127.22.254.61:36153)] connecting to peer
22:19:10.915 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.22.254.61:36153(127.22.254.61:36153)] Successfully connected to peer
22:19:10.915 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.22.254.62:37215(127.22.254.62:37215)] Successfully connected to peer
22:19:10.920 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.254.61:36153(127.22.254.61:36153)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:19:10.920 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.254.62:37215(127.22.254.62:37215)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:19:10.920 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.62:37215: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:19:10.920 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.61:36153: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:19:10.921 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153; will retry
22:19:10.921 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.22.254.62:37215(127.22.254.62:37215)] handling channelInactive
22:19:10.921 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.22.254.61:36153(127.22.254.61:36153)] handling channelInactive
22:19:10.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:19:10.923035 24241 negotiation.cc:337] Failed RPC negotiation. Trace:
22:19:10.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:10.916337 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:41406 (local address 127.22.254.62:37215)
22:19:10.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:10.919432 (+ 3095us) server_negotiation.cc:206] Beginning negotiation
22:19:10.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:10.919460 (+ 28us) server_negotiation.cc:399] Waiting for connection header
22:19:10.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:10.919504 (+ 44us) server_negotiation.cc:407] Connection header received
22:19:10.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:10.919630 (+ 126us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:19:10.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:10.919655 (+ 25us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:19:10.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:10.919856 (+ 201us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:19:10.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:10.921285 (+ 1429us) server_negotiation.cc:226] Negotiated authn=SASL
22:19:10.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:10.921914 (+ 629us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:41406: BlockingRecv error: recv got EOF from 127.0.0.1:41406 (error 108)
22:19:10.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":2712,"thread_start_us":5455,"threads_started":1}
22:19:10.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:19:10.923583 24240 negotiation.cc:337] Failed RPC negotiation. Trace:
22:19:10.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:10.916150 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:40022 (local address 127.22.254.61:36153)
22:19:10.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:10.919317 (+ 3167us) server_negotiation.cc:206] Beginning negotiation
22:19:10.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:10.919331 (+ 14us) server_negotiation.cc:399] Waiting for connection header
22:19:10.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:10.919364 (+ 33us) server_negotiation.cc:407] Connection header received
22:19:10.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:10.919460 (+ 96us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:19:10.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:10.919475 (+ 15us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:19:10.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:10.919623 (+ 148us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:19:10.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:10.919795 (+ 172us) server_negotiation.cc:226] Negotiated authn=SASL
22:19:10.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:10.922489 (+ 2694us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:40022: BlockingRecv error: recv got EOF from 127.0.0.1:40022 (error 108)
22:19:10.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":2808,"thread_start_us":2540,"threads_started":1}
22:19:11.413 [DEBUG - pool-70-thread-1] (Connection.java:748) [peer master-127.22.254.62:37215(127.22.254.62:37215)] connecting to peer
22:19:11.413 [DEBUG - pool-70-thread-1] (Connection.java:748) [peer master-127.22.254.61:36153(127.22.254.61:36153)] connecting to peer
22:19:11.414 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.22.254.62:37215(127.22.254.62:37215)] Successfully connected to peer
22:19:11.414 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.22.254.61:36153(127.22.254.61:36153)] Successfully connected to peer
22:19:11.416 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.254.62:37215(127.22.254.62:37215)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:19:11.416 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.62:37215: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:19:11.417 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.254.61:36153(127.22.254.61:36153)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:19:11.417 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.22.254.62:37215(127.22.254.62:37215)] handling channelInactive
22:19:11.417 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.61:36153: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:19:11.417 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.22.254.61:36153(127.22.254.61:36153)] handling channelInactive
22:19:11.417 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153; will retry
22:19:11.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:19:11.418323 24241 negotiation.cc:337] Failed RPC negotiation. Trace:
22:19:11.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:11.415179 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:43612 (local address 127.22.254.62:37215)
22:19:11.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:11.415674 (+ 495us) server_negotiation.cc:206] Beginning negotiation
22:19:11.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:11.415692 (+ 18us) server_negotiation.cc:399] Waiting for connection header
22:19:11.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:11.415725 (+ 33us) server_negotiation.cc:407] Connection header received
22:19:11.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:11.415826 (+ 101us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:19:11.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:11.415844 (+ 18us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:19:11.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:11.415996 (+ 152us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:19:11.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:11.416189 (+ 193us) server_negotiation.cc:226] Negotiated authn=SASL
22:19:11.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:11.417466 (+ 1277us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:43612: BlockingRecv error: recv got EOF from 127.0.0.1:43612 (error 108)
22:19:11.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":97}
22:19:11.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:19:11.418907 24240 negotiation.cc:337] Failed RPC negotiation. Trace:
22:19:11.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:11.415236 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:40558 (local address 127.22.254.61:36153)
22:19:11.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:11.415955 (+ 719us) server_negotiation.cc:206] Beginning negotiation
22:19:11.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:11.415988 (+ 33us) server_negotiation.cc:399] Waiting for connection header
22:19:11.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:11.416049 (+ 61us) server_negotiation.cc:407] Connection header received
22:19:11.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:11.416311 (+ 262us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:19:11.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:11.416339 (+ 28us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:19:11.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:11.416612 (+ 273us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:19:11.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:11.416840 (+ 228us) server_negotiation.cc:226] Negotiated authn=SASL
22:19:11.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:11.417901 (+ 1061us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:40558: BlockingRecv error: recv got EOF from 127.0.0.1:40558 (error 108)
22:19:11.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":177}
22:19:12.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 22:19:12 dist-test-slave-qzth krb5kdc[23555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790547552, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.254.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
22:19:12.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:19:12.711634 24083 init.cc:295] Successfully reacquired a new kerberos TGT
22:19:12.873 [DEBUG - pool-70-thread-1] (Connection.java:748) [peer master-127.22.254.62:37215(127.22.254.62:37215)] connecting to peer
22:19:12.874 [DEBUG - pool-70-thread-1] (Connection.java:748) [peer master-127.22.254.61:36153(127.22.254.61:36153)] connecting to peer
22:19:12.875 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.22.254.62:37215(127.22.254.62:37215)] Successfully connected to peer
22:19:12.875 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.22.254.61:36153(127.22.254.61:36153)] Successfully connected to peer
22:19:12.878 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.254.61:36153(127.22.254.61:36153)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:19:12.878 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.61:36153: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:19:12.879 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.22.254.61:36153(127.22.254.61:36153)] handling channelInactive
22:19:12.879 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.254.62:37215(127.22.254.62:37215)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:19:12.879 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.254.62:37215: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
22:19:12.880 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153; will retry
22:19:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:19:12.881471 24243 negotiation.cc:337] Failed RPC negotiation. Trace:
22:19:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:12.875958 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:40572 (local address 127.22.254.61:36153)
22:19:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:12.877427 (+ 1469us) server_negotiation.cc:206] Beginning negotiation
22:19:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:12.877453 (+ 26us) server_negotiation.cc:399] Waiting for connection header
22:19:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:12.877496 (+ 43us) server_negotiation.cc:407] Connection header received
22:19:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:12.877655 (+ 159us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:19:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:12.877678 (+ 23us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:19:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:12.877932 (+ 254us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:19:12.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:12.878984 (+ 1052us) server_negotiation.cc:226] Negotiated authn=SASL
22:19:12.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:12.880377 (+ 1393us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:40572: BlockingRecv error: recv got EOF from 127.0.0.1:40572 (error 108)
22:19:12.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":890,"thread_start_us":934,"threads_started":1}
22:19:12.882 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=29989), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [6ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [7ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [116ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [124ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [124ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT),org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [141ms] refreshing cache from master, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [146ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [147ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [159ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [159ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [185ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [193ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [195ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [195ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [205ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [205ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [211ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [212ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [227ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [227ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [230ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [230ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [245ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [256ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [259ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [259ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [341ms] refreshing cache from master, [341ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [342ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [342ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [346ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [346ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [356ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [357ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [501ms] refreshing cache from master, [501ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [502ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [502ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [506ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [506ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [507ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [507ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [721ms] refreshing cache from master, [721ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [721ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [722ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [724ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [725ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [726ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [727ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1161ms] refreshing cache from master, [1161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [1162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [1162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [1166ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1166ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1170ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [1170ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1881ms] refreshing cache from master, [1881ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [1882ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [1882ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [1887ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1887ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [1888ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1888ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [3141ms] refreshing cache from master, [3141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [3142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [3142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [3146ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [3146ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [3147ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [3147ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [5381ms] refreshing cache from master, [5381ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [5382ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [5382ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [5384ms] too many traces: truncated at 100 traces, deferred=Deferred@352225576(state=PENDING, result=null, callback=(continuation of Deferred@1273039085 after retry RPC after error@488580738) -> (continuation of Deferred@1967949085 after retry RPC after error@362538325) -> (continuation of Deferred@1451110763 after retry RPC after error@842085904) -> (continuation of Deferred@600058150 after retry RPC after error@1765790022) -> (continuation of Deferred@1324171363 after retry RPC after error@1968668184) -> (continuation of Deferred@2010702989 after retry RPC after error@1627045585) -> (continuation of Deferred@1144001781 after retry RPC after error@173610453) -> (continuation of Deferred@974480543 after retry RPC after error@254560570) -> (continuation of Deferred@1275032470 after retry RPC after error@164241537) -> (continuation of Deferred@418024635 after retry RPC after error@1969386245) -> (continuation of Deferred@1160891900 after retry RPC after error@497497075) -> (continuation of Deferred@1365572520 after retry RPC after error@930627802) -> (continuation of Deferred@2087532004 after retry RPC after error@1492232497) -> (continuation of Deferred@784248892 after retry RPC after error@1530223521) -> (continuation of Deferred@1323472694 after retry RPC after error@453974010) -> (continuation of Deferred@1089170423 after retry RPC after error@829610008) -> (continuation of Deferred@1811128871 after retry RPC after error@144754788) -> (continuation of Deferred@319794772 after retry RPC after error@874120757) -> (continuation of Deferred@1116629003 after retry RPC after error@1281588182) -> (continuation of Deferred@577769728 after retry RPC after error@1178566869) -> (continuation of Deferred@2000709161 after retry RPC after error@539631324) -> (continuation of Deferred@49341384 after retry RPC after error@955507307) -> (continuation of Deferred@1291233906 after retry RPC after error@1020957134), errback=(continuation of Deferred@1273039085 after retry RPC after error@488580738) -> (continuation of Deferred@1967949085 after retry RPC after error@362538325) -> (continuation of Deferred@1451110763 after retry RPC after error@842085904) -> (continuation of Deferred@600058150 after retry RPC after error@1765790022) -> (continuation of Deferred@1324171363 after retry RPC after error@1968668184) -> (continuation of Deferred@2010702989 after retry RPC after error@1627045585) -> (continuation of Deferred@1144001781 after retry RPC after error@173610453) -> (continuation of Deferred@974480543 after retry RPC after error@254560570) -> (continuation of Deferred@1275032470 after retry RPC after error@164241537) -> (continuation of Deferred@418024635 after retry RPC after error@1969386245) -> (continuation of Deferred@1160891900 after retry RPC after error@497497075) -> (continuation of Deferred@1365572520 after retry RPC after error@930627802) -> (continuation of Deferred@2087532004 after retry RPC after error@1492232497) -> (continuation of Deferred@784248892 after retry RPC after error@1530223521) -> (continuation of Deferred@1323472694 after retry RPC after error@453974010) -> (continuation of Deferred@1089170423 after retry RPC after error@829610008) -> (continuation of Deferred@1811128871 after retry RPC after error@144754788) -> (continuation of Deferred@319794772 after retry RPC after error@874120757) -> (continuation of Deferred@1116629003 after retry RPC after error@1281588182) -> (continuation of Deferred@577769728 after retry RPC after error@1178566869) -> (continuation of Deferred@2000709161 after retry RPC after error@539631324) -> (continuation of Deferred@49341384 after retry RPC after error@955507307) -> (continuation of Deferred@1291233906 after retry RPC after error@1020957134)))
22:19:12.884 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.22.254.61:36153(127.22.254.61:36153)] disconnecting while in state TERMINATED
22:19:12.884 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.22.254.60:44023(127.22.254.60:44023)] disconnecting while in state READY
22:19:12.884 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.22.254.62:37215(127.22.254.62:37215)] disconnecting while in state TERMINATED
22:19:12.887 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
22:19:12.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 22:19:12.888332 24244 negotiation.cc:337] Failed RPC negotiation. Trace:
22:19:12.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:12.876678 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:43618 (local address 127.22.254.62:37215)
22:19:12.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:12.878618 (+ 1940us) server_negotiation.cc:206] Beginning negotiation
22:19:12.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:12.878647 (+ 29us) server_negotiation.cc:399] Waiting for connection header
22:19:12.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:12.878748 (+ 101us) server_negotiation.cc:407] Connection header received
22:19:12.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:12.878911 (+ 163us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
22:19:12.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:12.878940 (+ 29us) server_negotiation.cc:461] Received NEGOTIATE request from client
22:19:12.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:12.879262 (+ 322us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
22:19:12.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:12.879489 (+ 227us) server_negotiation.cc:226] Negotiated authn=SASL
22:19:12.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 22:19:12.887171 (+ 7682us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:43618: BlockingRecv error: recv got EOF from 127.0.0.1:43618 (error 108)
22:19:12.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":1448,"thread_start_us":1424,"threads_started":1}
22:19:12.889 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
22:19:12.890 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.22.254.62:37215(127.22.254.62:37215)] handling channelInactive
22:19:12.890 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.22.254.60:44023(127.22.254.60:44023)] handling channelInactive
22:19:12.890 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.254.60:44023(127.22.254.60:44023)] cleaning up while in state READY due to: connection closed
22:19:12.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:19:12.899667 23544 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu with pid 23878
22:19:12.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:19:12.927071 23544 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu with pid 24042
22:19:12.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:19:12.951067 23544 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu with pid 23950
22:19:14.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T22:19:14Z chronyd exiting
22:19:15.150 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
22:19:15.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:19:15.151376 22820 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu with pid 23100
22:19:15.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:19:15.176837 22820 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu with pid 23258
22:19:15.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:19:15.204046 22820 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu with pid 23403
22:19:15.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:19:15.233529 22820 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu with pid 22832
22:19:15.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:19:15.263794 22820 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu with pid 22902
22:19:15.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 22:19:15.296315 22820 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskyO6VHJ/build/tsan/bin/kudu with pid 22977
22:19:16.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T22:19:16Z chronyd exiting
22:19:17.447 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectExpires: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=29989), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [6ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [7ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [116ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [124ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [124ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT),org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [141ms] refreshing cache from master, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [146ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [147ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [159ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [159ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [185ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [193ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [195ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [195ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [205ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [205ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [211ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [212ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [227ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [227ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [230ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [230ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [245ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [256ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [259ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [259ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [341ms] refreshing cache from master, [341ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [342ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [342ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [346ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [346ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [356ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [357ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [501ms] refreshing cache from master, [501ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [502ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [502ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [506ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [506ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [507ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [507ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [721ms] refreshing cache from master, [721ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [721ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [722ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [724ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [725ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [726ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [727ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1161ms] refreshing cache from master, [1161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [1162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [1162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [1166ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1166ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1170ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [1170ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1881ms] refreshing cache from master, [1881ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [1882ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [1882ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [1887ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1887ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [1888ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1888ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [3141ms] refreshing cache from master, [3141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [3142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [3142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [3146ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [3146ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [3147ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [3147ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [5381ms] refreshing cache from master, [5381ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [5382ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [5382ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [5384ms] too many traces: truncated at 100 traces, deferred=Deferred@352225576(state=PENDING, result=null, callback=(continuation of Deferred@1273039085 after retry RPC after error@488580738) -> (continuation of Deferred@1967949085 after retry RPC after error@362538325) -> (continuation of Deferred@1451110763 after retry RPC after error@842085904) -> (continuation of Deferred@600058150 after retry RPC after error@1765790022) -> (continuation of Deferred@1324171363 after retry RPC after error@1968668184) -> (continuation of Deferred@2010702989 after retry RPC after error@1627045585) -> (continuation of Deferred@1144001781 after retry RPC after error@173610453) -> (continuation of Deferred@974480543 after retry RPC after error@254560570) -> (continuation of Deferred@1275032470 after retry RPC after error@164241537) -> (continuation of Deferred@418024635 after retry RPC after error@1969386245) -> (continuation of Deferred@1160891900 after retry RPC after error@497497075) -> (continuation of Deferred@1365572520 after retry RPC after error@930627802) -> (continuation of Deferred@2087532004 after retry RPC after error@1492232497) -> (continuation of Deferred@784248892 after retry RPC after error@1530223521) -> (continuation of Deferred@1323472694 after retry RPC after error@453974010) -> (continuation of Deferred@1089170423 after retry RPC after error@829610008) -> (continuation of Deferred@1811128871 after retry RPC after error@144754788) -> (continuation of Deferred@319794772 after retry RPC after error@874120757) -> (continuation of Deferred@1116629003 after retry RPC after error@1281588182) -> (continuation of Deferred@577769728 after retry RPC after error@1178566869) -> (continuation of Deferred@2000709161 after retry RPC after error@539631324) -> (continuation of Deferred@49341384 after retry RPC after error@955507307) -> (continuation of Deferred@1291233906 after retry RPC after error@1020957134), errback=(continuation of Deferred@1273039085 after retry RPC after error@488580738) -> (continuation of Deferred@1967949085 after retry RPC after error@362538325) -> (continuation of Deferred@1451110763 after retry RPC after error@842085904) -> (continuation of Deferred@600058150 after retry RPC after error@1765790022) -> (continuation of Deferred@1324171363 after retry RPC after error@1968668184) -> (continuation of Deferred@2010702989 after retry RPC after error@1627045585) -> (continuation of Deferred@1144001781 after retry RPC after error@173610453) -> (continuation of Deferred@974480543 after retry RPC after error@254560570) -> (continuation of Deferred@1275032470 after retry RPC after error@164241537) -> (continuation of Deferred@418024635 after retry RPC after error@1969386245) -> (continuation of Deferred@1160891900 after retry RPC after error@497497075) -> (continuation of Deferred@1365572520 after retry RPC after error@930627802) -> (continuation of Deferred@2087532004 after retry RPC after error@1492232497) -> (continuation of Deferred@784248892 after retry RPC after error@1530223521) -> (continuation of Deferred@1323472694 after retry RPC after error@453974010) -> (continuation of Deferred@1089170423 after retry RPC after error@829610008) -> (continuation of Deferred@1811128871 after retry RPC after error@144754788) -> (continuation of Deferred@319794772 after retry RPC after error@874120757) -> (continuation of Deferred@1116629003 after retry RPC after error@1281588182) -> (continuation of Deferred@577769728 after retry RPC after error@1178566869) -> (continuation of Deferred@2000709161 after retry RPC after error@539631324) -> (continuation of Deferred@49341384 after retry RPC after error@955507307) -> (continuation of Deferred@1291233906 after retry RPC after error@1020957134)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386) ~[test/:?]
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectExpires(TestSecurity.java:412) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565) ~[main/:?]
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=29989), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [6ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [7ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [116ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [124ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [124ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT),org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [141ms] refreshing cache from master, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [146ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [147ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [159ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [159ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [185ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [193ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [195ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [195ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [205ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [205ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [211ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [212ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [227ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [227ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [230ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [230ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [245ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [256ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [259ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [259ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [341ms] refreshing cache from master, [341ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [342ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [342ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [346ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [346ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [356ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [357ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [501ms] refreshing cache from master, [501ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [502ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [502ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [506ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [506ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [507ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [507ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [721ms] refreshing cache from master, [721ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [721ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [722ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [724ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [725ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [726ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [727ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1161ms] refreshing cache from master, [1161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [1162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [1162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [1166ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1166ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1170ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [1170ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1881ms] refreshing cache from master, [1881ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [1882ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [1882ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [1887ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1887ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [1888ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1888ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [3141ms] refreshing cache from master, [3141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [3142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [3142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [3146ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [3146ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [3147ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [3147ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [5381ms] refreshing cache from master, [5381ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [5382ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [5382ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [5384ms] too many traces: truncated at 100 traces, deferred=Deferred@352225576(state=PENDING, result=null, callback=(continuation of Deferred@1273039085 after retry RPC after error@488580738) -> (continuation of Deferred@1967949085 after retry RPC after error@362538325) -> (continuation of Deferred@1451110763 after retry RPC after error@842085904) -> (continuation of Deferred@600058150 after retry RPC after error@1765790022) -> (continuation of Deferred@1324171363 after retry RPC after error@1968668184) -> (continuation of Deferred@2010702989 after retry RPC after error@1627045585) -> (continuation of Deferred@1144001781 after retry RPC after error@173610453) -> (continuation of Deferred@974480543 after retry RPC after error@254560570) -> (continuation of Deferred@1275032470 after retry RPC after error@164241537) -> (continuation of Deferred@418024635 after retry RPC after error@1969386245) -> (continuation of Deferred@1160891900 after retry RPC after error@497497075) -> (continuation of Deferred@1365572520 after retry RPC after error@930627802) -> (continuation of Deferred@2087532004 after retry RPC after error@1492232497) -> (continuation of Deferred@784248892 after retry RPC after error@1530223521) -> (continuation of Deferred@1323472694 after retry RPC after error@453974010) -> (continuation of Deferred@1089170423 after retry RPC after error@829610008) -> (continuation of Deferred@1811128871 after retry RPC after error@144754788) -> (continuation of Deferred@319794772 after retry RPC after error@874120757) -> (continuation of Deferred@1116629003 after retry RPC after error@1281588182) -> (continuation of Deferred@577769728 after retry RPC after error@1178566869) -> (continuation of Deferred@2000709161 after retry RPC after error@539631324) -> (continuation of Deferred@49341384 after retry RPC after error@955507307) -> (continuation of Deferred@1291233906 after retry RPC after error@1020957134), errback=(continuation of Deferred@1273039085 after retry RPC after error@488580738) -> (continuation of Deferred@1967949085 after retry RPC after error@362538325) -> (continuation of Deferred@1451110763 after retry RPC after error@842085904) -> (continuation of Deferred@600058150 after retry RPC after error@1765790022) -> (continuation of Deferred@1324171363 after retry RPC after error@1968668184) -> (continuation of Deferred@2010702989 after retry RPC after error@1627045585) -> (continuation of Deferred@1144001781 after retry RPC after error@173610453) -> (continuation of Deferred@974480543 after retry RPC after error@254560570) -> (continuation of Deferred@1275032470 after retry RPC after error@164241537) -> (continuation of Deferred@418024635 after retry RPC after error@1969386245) -> (continuation of Deferred@1160891900 after retry RPC after error@497497075) -> (continuation of Deferred@1365572520 after retry RPC after error@930627802) -> (continuation of Deferred@2087532004 after retry RPC after error@1492232497) -> (continuation of Deferred@784248892 after retry RPC after error@1530223521) -> (continuation of Deferred@1323472694 after retry RPC after error@453974010) -> (continuation of Deferred@1089170423 after retry RPC after error@829610008) -> (continuation of Deferred@1811128871 after retry RPC after error@144754788) -> (continuation of Deferred@319794772 after retry RPC after error@874120757) -> (continuation of Deferred@1116629003 after retry RPC after error@1281588182) -> (continuation of Deferred@577769728 after retry RPC after error@1178566869) -> (continuation of Deferred@2000709161 after retry RPC after error@539631324) -> (continuation of Deferred@49341384 after retry RPC after error@955507307) -> (continuation of Deferred@1291233906 after retry RPC after error@1020957134)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386)
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40)
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectExpires(TestSecurity.java:412)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
... 30 more
22:19:18.401 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectExpires: giving up after 1 attempts
22:19:18.401 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectExpires: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=29989), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [6ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [7ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [116ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [124ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [124ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT),org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [141ms] refreshing cache from master, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [146ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [147ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [159ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [159ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [185ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [193ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [195ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [195ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [205ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [205ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [211ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [212ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [227ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [227ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [230ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [230ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [245ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [256ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [259ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [259ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [341ms] refreshing cache from master, [341ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [342ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [342ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [346ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [346ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [356ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [357ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [501ms] refreshing cache from master, [501ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [502ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [502ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [506ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [506ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [507ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [507ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [721ms] refreshing cache from master, [721ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [721ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [722ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [724ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [725ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [726ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [727ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1161ms] refreshing cache from master, [1161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [1162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [1162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [1166ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1166ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1170ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [1170ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1881ms] refreshing cache from master, [1881ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [1882ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [1882ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [1887ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1887ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [1888ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1888ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [3141ms] refreshing cache from master, [3141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [3142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [3142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [3146ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [3146ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [3147ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [3147ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [5381ms] refreshing cache from master, [5381ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [5382ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [5382ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [5384ms] too many traces: truncated at 100 traces, deferred=Deferred@352225576(state=PENDING, result=null, callback=(continuation of Deferred@1273039085 after retry RPC after error@488580738) -> (continuation of Deferred@1967949085 after retry RPC after error@362538325) -> (continuation of Deferred@1451110763 after retry RPC after error@842085904) -> (continuation of Deferred@600058150 after retry RPC after error@1765790022) -> (continuation of Deferred@1324171363 after retry RPC after error@1968668184) -> (continuation of Deferred@2010702989 after retry RPC after error@1627045585) -> (continuation of Deferred@1144001781 after retry RPC after error@173610453) -> (continuation of Deferred@974480543 after retry RPC after error@254560570) -> (continuation of Deferred@1275032470 after retry RPC after error@164241537) -> (continuation of Deferred@418024635 after retry RPC after error@1969386245) -> (continuation of Deferred@1160891900 after retry RPC after error@497497075) -> (continuation of Deferred@1365572520 after retry RPC after error@930627802) -> (continuation of Deferred@2087532004 after retry RPC after error@1492232497) -> (continuation of Deferred@784248892 after retry RPC after error@1530223521) -> (continuation of Deferred@1323472694 after retry RPC after error@453974010) -> (continuation of Deferred@1089170423 after retry RPC after error@829610008) -> (continuation of Deferred@1811128871 after retry RPC after error@144754788) -> (continuation of Deferred@319794772 after retry RPC after error@874120757) -> (continuation of Deferred@1116629003 after retry RPC after error@1281588182) -> (continuation of Deferred@577769728 after retry RPC after error@1178566869) -> (continuation of Deferred@2000709161 after retry RPC after error@539631324) -> (continuation of Deferred@49341384 after retry RPC after error@955507307) -> (continuation of Deferred@1291233906 after retry RPC after error@1020957134), errback=(continuation of Deferred@1273039085 after retry RPC after error@488580738) -> (continuation of Deferred@1967949085 after retry RPC after error@362538325) -> (continuation of Deferred@1451110763 after retry RPC after error@842085904) -> (continuation of Deferred@600058150 after retry RPC after error@1765790022) -> (continuation of Deferred@1324171363 after retry RPC after error@1968668184) -> (continuation of Deferred@2010702989 after retry RPC after error@1627045585) -> (continuation of Deferred@1144001781 after retry RPC after error@173610453) -> (continuation of Deferred@974480543 after retry RPC after error@254560570) -> (continuation of Deferred@1275032470 after retry RPC after error@164241537) -> (continuation of Deferred@418024635 after retry RPC after error@1969386245) -> (continuation of Deferred@1160891900 after retry RPC after error@497497075) -> (continuation of Deferred@1365572520 after retry RPC after error@930627802) -> (continuation of Deferred@2087532004 after retry RPC after error@1492232497) -> (continuation of Deferred@784248892 after retry RPC after error@1530223521) -> (continuation of Deferred@1323472694 after retry RPC after error@453974010) -> (continuation of Deferred@1089170423 after retry RPC after error@829610008) -> (continuation of Deferred@1811128871 after retry RPC after error@144754788) -> (continuation of Deferred@319794772 after retry RPC after error@874120757) -> (continuation of Deferred@1116629003 after retry RPC after error@1281588182) -> (continuation of Deferred@577769728 after retry RPC after error@1178566869) -> (continuation of Deferred@2000709161 after retry RPC after error@539631324) -> (continuation of Deferred@49341384 after retry RPC after error@955507307) -> (continuation of Deferred@1291233906 after retry RPC after error@1020957134)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386) ~[test/:?]
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectExpires(TestSecurity.java:412) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565) ~[main/:?]
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=29989), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [6ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [7ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [116ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [122ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [124ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [124ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT),org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [141ms] refreshing cache from master, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [146ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [147ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [159ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [159ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [185ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [193ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [195ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [195ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [205ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [205ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [211ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [212ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [227ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [227ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [230ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [230ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [245ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [256ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [259ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [259ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [341ms] refreshing cache from master, [341ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [342ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [342ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [346ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [346ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [356ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [357ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [501ms] refreshing cache from master, [501ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [502ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [502ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [506ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [506ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [507ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [507ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [721ms] refreshing cache from master, [721ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [721ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [722ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [724ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [725ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [726ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [727ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1161ms] refreshing cache from master, [1161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [1162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [1162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [1166ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1166ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1170ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [1170ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1881ms] refreshing cache from master, [1881ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [1882ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [1882ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [1887ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1887ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [1888ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1888ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [3141ms] refreshing cache from master, [3141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [3142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [3142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [3146ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.60:44023: OK, [3146ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.62:37215: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [3147ms] Sub RPC ConnectToMaster: received response from server master-127.22.254.61:36153: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [3147ms] delaying RPC due to: Service unavailable: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [5381ms] refreshing cache from master, [5381ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.62:37215, [5382ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.60:44023, [5382ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.254.61:36153, [5384ms] too many traces: truncated at 100 traces, deferred=Deferred@352225576(state=PENDING, result=null, callback=(continuation of Deferred@1273039085 after retry RPC after error@488580738) -> (continuation of Deferred@1967949085 after retry RPC after error@362538325) -> (continuation of Deferred@1451110763 after retry RPC after error@842085904) -> (continuation of Deferred@600058150 after retry RPC after error@1765790022) -> (continuation of Deferred@1324171363 after retry RPC after error@1968668184) -> (continuation of Deferred@2010702989 after retry RPC after error@1627045585) -> (continuation of Deferred@1144001781 after retry RPC after error@173610453) -> (continuation of Deferred@974480543 after retry RPC after error@254560570) -> (continuation of Deferred@1275032470 after retry RPC after error@164241537) -> (continuation of Deferred@418024635 after retry RPC after error@1969386245) -> (continuation of Deferred@1160891900 after retry RPC after error@497497075) -> (continuation of Deferred@1365572520 after retry RPC after error@930627802) -> (continuation of Deferred@2087532004 after retry RPC after error@1492232497) -> (continuation of Deferred@784248892 after retry RPC after error@1530223521) -> (continuation of Deferred@1323472694 after retry RPC after error@453974010) -> (continuation of Deferred@1089170423 after retry RPC after error@829610008) -> (continuation of Deferred@1811128871 after retry RPC after error@144754788) -> (continuation of Deferred@319794772 after retry RPC after error@874120757) -> (continuation of Deferred@1116629003 after retry RPC after error@1281588182) -> (continuation of Deferred@577769728 after retry RPC after error@1178566869) -> (continuation of Deferred@2000709161 after retry RPC after error@539631324) -> (continuation of Deferred@49341384 after retry RPC after error@955507307) -> (continuation of Deferred@1291233906 after retry RPC after error@1020957134), errback=(continuation of Deferred@1273039085 after retry RPC after error@488580738) -> (continuation of Deferred@1967949085 after retry RPC after error@362538325) -> (continuation of Deferred@1451110763 after retry RPC after error@842085904) -> (continuation of Deferred@600058150 after retry RPC after error@1765790022) -> (continuation of Deferred@1324171363 after retry RPC after error@1968668184) -> (continuation of Deferred@2010702989 after retry RPC after error@1627045585) -> (continuation of Deferred@1144001781 after retry RPC after error@173610453) -> (continuation of Deferred@974480543 after retry RPC after error@254560570) -> (continuation of Deferred@1275032470 after retry RPC after error@164241537) -> (continuation of Deferred@418024635 after retry RPC after error@1969386245) -> (continuation of Deferred@1160891900 after retry RPC after error@497497075) -> (continuation of Deferred@1365572520 after retry RPC after error@930627802) -> (continuation of Deferred@2087532004 after retry RPC after error@1492232497) -> (continuation of Deferred@784248892 after retry RPC after error@1530223521) -> (continuation of Deferred@1323472694 after retry RPC after error@453974010) -> (continuation of Deferred@1089170423 after retry RPC after error@829610008) -> (continuation of Deferred@1811128871 after retry RPC after error@144754788) -> (continuation of Deferred@319794772 after retry RPC after error@874120757) -> (continuation of Deferred@1116629003 after retry RPC after error@1281588182) -> (continuation of Deferred@577769728 after retry RPC after error@1178566869) -> (continuation of Deferred@2000709161 after retry RPC after error@539631324) -> (continuation of Deferred@49341384 after retry RPC after error@955507307) -> (continuation of Deferred@1291233906 after retry RPC after error@1020957134)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386)
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40)
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectExpires(TestSecurity.java:412)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.22.254.62:37215,127.22.254.60:44023,127.22.254.61:36153) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available,org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
... 30 more