20:37:52.430 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
20:37:52.443 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task0rSsIl/test-tmp
20:37:52.446 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
20:37:52.449 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task0rSsIl/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
20:37:52.719 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
20:37:52.885 [DEBUG - main] (MiniKuduCluster.java:195) Response:
20:37:52.886 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
20:37:52.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:52.890431 10980 mini_oidc.cc:152] Starting JWKS server
20:37:52.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:52.895512 10980 webserver.cc:460] Webserver started at http://127.0.0.1:46757/ using document root <none> and password file <none>
20:37:52.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:52.896026 10980 mini_oidc.cc:189] Starting OIDC Discovery server
20:37:52.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:52.898509 10980 webserver.cc:460] Webserver started at http://127.0.0.1:36007/ using document root <none> and password file <none>
20:37:52.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T20:37:52Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
20:37:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T20:37:52Z Disabled control of system clock
20:37:52.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:52.951890 10980 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0rSsIl/build/tsan/bin/kudu
20:37:52.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0rSsIl/build/tsan/bin/kudu
20:37:52.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-0/wal
20:37:52.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-0/data
20:37:52.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:37:52.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:37:52.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:37:52.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:37:52.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:37:52.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:37:52.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-0/logs
20:37:52.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-0/data/info.pb
20:37:52.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:37:52.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:37:52.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:37:52.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:37:52.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:37:52.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:37:52.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:37:52.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:37:52.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:37:52.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:37:52.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:37:52.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:37:52.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.185.62:46507
20:37:52.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.185.62
20:37:52.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:37:52.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.185.20:34965
20:37:52.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:37:52.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:37:52.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:37:52.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:37:52.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.185.62:46507,127.10.185.61:35973,127.10.185.60:34451
20:37:52.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:37:52.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46757/jwks.json with env {}
20:37:53.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:53.254978 10992 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:37:53.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:53.255658 10992 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:37:53.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:53.256309 10992 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:37:53.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:53.286757 10992 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:37:53.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:53.287065 10992 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:37:53.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:53.287254 10992 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:37:53.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:53.287441 10992 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:37:53.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:53.287715 10992 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:37:53.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:53.287945 10992 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46757/jwks.json
20:37:53.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:53.323618 10992 master_runner.cc:386] Master server non-default flags:
20:37:53.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:37:53.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.185.20:34965
20:37:53.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:37:53.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:37:53.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-0/data
20:37:53.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-0/wal
20:37:53.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:37:53.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.185.62:46507,127.10.185.61:35973,127.10.185.60:34451
20:37:53.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:37:53.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:37:53.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:37:53.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.185.62:46507
20:37:53.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:37:53.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:37:53.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:37:53.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46757/jwks.json
20:37:53.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:37:53.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:37:53.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-0/data/info.pb
20:37:53.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.185.62
20:37:53.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:37:53.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:37:53.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:37:53.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:37:53.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:37:53.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:37:53.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-0/logs
20:37:53.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:37:53.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:37:53.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:37:53.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:37:53.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:37:53.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:37:53.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:37:53.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 18:34:45 UTC on ccycloud.quasar-elnkgk.root.comops.site
20:37:53.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677568
20:37:53.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:37:53.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:53.325223 10992 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:37:53.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:53.327668 10992 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:37:53.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:53.355091 10998 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:37:54.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:53.355216 11001 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:37:54.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:53.355611 10999 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:37:54.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:53.357339 10992 server_base.cc:1034] running on GCE node
20:37:54.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:54.619874 10992 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:37:54.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:54.623627 10992 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:37:54.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:54.625087 10992 hybrid_clock.cc:648] HybridClock initialized: now 1790627874625052 us; error 53 us; skew 500 ppm
20:37:54.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:54.625919 10992 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:37:54.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:54.638949 10992 webserver.cc:460] Webserver started at http://127.10.185.62:40357/ using document root <none> and password file <none>
20:37:54.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:54.639997 10992 fs_manager.cc:359] Metadata directory not provided
20:37:54.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:54.640183 10992 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:37:54.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:54.640698 10992 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:37:54.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:54.645834 10992 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-0/data/instance:
20:37:54.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d641128def46413893222c5b2bd5d23b"
20:37:54.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 20:37:54 on dist-test-slave-qb38"
20:37:54.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:54.647290 10992 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-0/wal/instance:
20:37:54.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d641128def46413893222c5b2bd5d23b"
20:37:54.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 20:37:54 on dist-test-slave-qb38"
20:37:54.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:54.654783 10992 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
20:37:54.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:54.661311 11008 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:37:54.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:54.662616 10992 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.006s sys 0.000s
20:37:54.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:54.662977 10992 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-0/data,/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-0/wal
20:37:54.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d641128def46413893222c5b2bd5d23b"
20:37:54.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 20:37:54 on dist-test-slave-qb38"
20:37:54.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:54.663439 10992 fs_report.cc:383] FS layout report
20:37:54.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:37:54.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-0/wal
20:37:54.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-0/wal
20:37:54.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-0/data/data
20:37:54.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:37:54.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:37:54.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:37:54.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:37:54.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:37:54.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:37:54.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:37:54.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:37:54.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:37:54.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:37:54.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:37:54.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:54.703994 10992 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:37:54.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:54.705845 10992 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:37:54.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:54.706455 10992 kserver.cc:163] Server-wide thread pool size limit: 3276
20:37:54.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:54.810956 10992 rpc_server.cc:307] RPC server started. Bound to: 127.10.185.62:46507
20:37:54.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:54.811259 11059 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.185.62:46507 every 8 connection(s)
20:37:54.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:54.814206 10992 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-0/data/info.pb
20:37:54.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:54.819418 10980 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0rSsIl/build/tsan/bin/kudu as pid 10992
20:37:54.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:54.820274 10980 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-0/wal/instance
20:37:54.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:54.827420 10980 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0rSsIl/build/tsan/bin/kudu
20:37:54.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0rSsIl/build/tsan/bin/kudu
20:37:54.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-1/wal
20:37:54.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-1/data
20:37:54.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:37:54.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:37:54.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:37:54.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:37:54.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:37:54.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:37:54.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-1/logs
20:37:54.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-1/data/info.pb
20:37:54.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:37:54.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:37:54.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:37:54.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:37:54.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:37:54.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:37:54.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:37:54.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:37:54.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:37:54.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:37:54.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:37:54.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:37:54.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.185.61:35973
20:37:54.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.185.61
20:37:54.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:37:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.185.20:34965
20:37:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:37:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:37:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:37:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:37:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.185.62:46507,127.10.185.61:35973,127.10.185.60:34451
20:37:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:37:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46757/jwks.json with env {}
20:37:54.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:54.830477 11060 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:37:54.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:54.846340 11060 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.185.62" port: 46507 } has no permanent_uuid. Determining permanent_uuid...
20:37:54.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:54.880702 11060 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.185.61" port: 35973 } has no permanent_uuid. Determining permanent_uuid...
20:37:54.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:54.883275 11010 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.185.61:35973: connect: Connection refused (error 111)
20:37:54.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:54.887494 11060 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.185.61:35973: Network error: Client connection negotiation failed: client connection to 127.10.185.61:35973: connect: Connection refused (error 111)
20:37:54.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:54.937036 11060 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.185.61" port: 35973 } attempt: 1
20:37:54.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:54.941823 11060 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.185.61:35973: Network error: Client connection negotiation failed: client connection to 127.10.185.61:35973: connect: Connection refused (error 111)
20:37:55.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:55.010459 11060 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.185.61" port: 35973 } attempt: 2
20:37:55.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:55.015759 11060 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.185.61:35973: Network error: Client connection negotiation failed: client connection to 127.10.185.61:35973: connect: Connection refused (error 111)
20:37:55.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:55.107241 11060 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.185.61" port: 35973 } attempt: 3
20:37:55.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:55.111371 11060 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.185.61:35973: Network error: Client connection negotiation failed: client connection to 127.10.185.61:35973: connect: Connection refused (error 111)
20:37:55.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:55.176715 11062 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:37:55.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:55.177276 11062 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:37:55.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:55.177822 11062 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:37:55.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:55.208971 11062 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:37:55.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:55.209247 11062 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:37:55.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:55.209462 11062 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:37:55.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:55.209640 11062 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:37:55.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:55.209810 11062 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:37:55.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:55.209954 11062 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46757/jwks.json
20:37:55.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:55.244484 11062 master_runner.cc:386] Master server non-default flags:
20:37:55.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:37:55.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.185.20:34965
20:37:55.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:37:55.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:37:55.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-1/data
20:37:55.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-1/wal
20:37:55.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:37:55.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.185.62:46507,127.10.185.61:35973,127.10.185.60:34451
20:37:55.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:37:55.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:37:55.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:37:55.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.185.61:35973
20:37:55.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:37:55.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:37:55.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:37:55.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46757/jwks.json
20:37:55.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:37:55.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:37:55.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-1/data/info.pb
20:37:55.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.185.61
20:37:55.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:37:55.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:37:55.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:37:55.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:37:55.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:37:55.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:37:55.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-1/logs
20:37:55.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:37:55.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:37:55.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:37:55.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:37:55.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:37:55.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:37:55.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:37:55.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 18:34:45 UTC on ccycloud.quasar-elnkgk.root.comops.site
20:37:55.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677568
20:37:55.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:37:55.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:55.245801 11062 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:37:55.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:55.247504 11062 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:37:55.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:55.254760 11060 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.185.61" port: 35973 } attempt: 4
20:37:55.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:55.259161 11060 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.185.61:35973: Network error: Client connection negotiation failed: client connection to 127.10.185.61:35973: connect: Connection refused (error 111)
20:37:55.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:55.271292 11072 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:37:55.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:55.558652 11060 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.185.61" port: 35973 } attempt: 5
20:37:55.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:55.563892 11060 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.185.61:35973: Network error: Client connection negotiation failed: client connection to 127.10.185.61:35973: connect: Connection refused (error 111)
20:37:56.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:56.111335 11060 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.185.61" port: 35973 } attempt: 6
20:37:56.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:56.123152 11060 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.185.61:35973: Network error: Client connection negotiation failed: client connection to 127.10.185.61:35973: connect: Connection refused (error 111)
20:37:56.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:55.271432 11071 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:37:56.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:55.272022 11074 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:37:56.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:56.511417 11073 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
20:37:56.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:56.511988 11062 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:37:56.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:56.516269 11062 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:37:56.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:56.518955 11062 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:37:56.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:56.520327 11062 hybrid_clock.cc:648] HybridClock initialized: now 1790627876520317 us; error 55 us; skew 500 ppm
20:37:56.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:56.521260 11062 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:37:56.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:56.528590 11062 webserver.cc:460] Webserver started at http://127.10.185.61:46783/ using document root <none> and password file <none>
20:37:56.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:56.529708 11062 fs_manager.cc:359] Metadata directory not provided
20:37:56.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:56.529944 11062 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:37:56.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:56.530555 11062 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:37:56.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:56.536581 11062 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-1/data/instance:
20:37:56.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1d39e2e4ef6041af8b602bbd15db4dd6"
20:37:56.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 20:37:56 on dist-test-slave-qb38"
20:37:56.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:56.537858 11062 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-1/wal/instance:
20:37:56.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1d39e2e4ef6041af8b602bbd15db4dd6"
20:37:56.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 20:37:56 on dist-test-slave-qb38"
20:37:56.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:56.548300 11062 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.004s sys 0.004s
20:37:56.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:56.556357 11084 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:37:56.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:56.557437 11062 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.004s
20:37:56.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:56.557796 11062 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-1/data,/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-1/wal
20:37:56.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1d39e2e4ef6041af8b602bbd15db4dd6"
20:37:56.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 20:37:56 on dist-test-slave-qb38"
20:37:56.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:56.558204 11062 fs_report.cc:383] FS layout report
20:37:56.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:37:56.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-1/wal
20:37:56.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-1/wal
20:37:56.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-1/data/data
20:37:56.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:37:56.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:37:56.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:37:56.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:37:56.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:37:56.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:37:56.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:37:56.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:37:56.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:37:56.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:37:56.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:37:56.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:56.614655 11062 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:37:56.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:56.616240 11062 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:37:56.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:56.616776 11062 kserver.cc:163] Server-wide thread pool size limit: 3276
20:37:56.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:56.697031 11062 rpc_server.cc:307] RPC server started. Bound to: 127.10.185.61:35973
20:37:56.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:56.697120 11135 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.185.61:35973 every 8 connection(s)
20:37:56.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:56.700238 11062 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-1/data/info.pb
20:37:56.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:56.705955 11136 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:37:56.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:56.707159 10980 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0rSsIl/build/tsan/bin/kudu as pid 11062
20:37:56.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:56.707600 10980 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-1/wal/instance
20:37:56.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:56.709745 10980 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0rSsIl/build/tsan/bin/kudu
20:37:56.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0rSsIl/build/tsan/bin/kudu
20:37:56.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-2/wal
20:37:56.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-2/data
20:37:56.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:37:56.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:37:56.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:37:56.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:37:56.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:37:56.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:37:56.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-2/logs
20:37:56.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-2/data/info.pb
20:37:56.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:37:56.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:37:56.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:37:56.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:37:56.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:37:56.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:37:56.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:37:56.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:37:56.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:37:56.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:37:56.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:37:56.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:37:56.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.185.60:34451
20:37:56.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.185.60
20:37:56.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:37:56.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.185.20:34965
20:37:56.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:37:56.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:37:56.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:37:56.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:37:56.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.185.62:46507,127.10.185.61:35973,127.10.185.60:34451
20:37:56.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:37:56.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46757/jwks.json with env {}
20:37:56.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:56.719949 11136 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.185.62" port: 46507 } has no permanent_uuid. Determining permanent_uuid...
20:37:56.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:56.747496 11136 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.185.61" port: 35973 } has no permanent_uuid. Determining permanent_uuid...
20:37:56.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:56.764727 11136 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.185.60" port: 34451 } has no permanent_uuid. Determining permanent_uuid...
20:37:56.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:56.767194 11086 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.185.60:34451: connect: Connection refused (error 111)
20:37:56.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:56.773751 11136 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.185.60:34451: Network error: Client connection negotiation failed: client connection to 127.10.185.60:34451: connect: Connection refused (error 111)
20:37:56.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:56.823290 11136 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.185.60" port: 34451 } attempt: 1
20:37:56.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:56.827865 11136 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.185.60:34451: Network error: Client connection negotiation failed: client connection to 127.10.185.60:34451: connect: Connection refused (error 111)
20:37:56.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:56.896270 11136 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.185.60" port: 34451 } attempt: 2
20:37:56.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:56.900902 11136 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.185.60:34451: Network error: Client connection negotiation failed: client connection to 127.10.185.60:34451: connect: Connection refused (error 111)
20:37:56.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:56.992326 11136 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.185.60" port: 34451 } attempt: 3
20:37:56.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:56.996789 11136 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.185.60:34451: Network error: Client connection negotiation failed: client connection to 127.10.185.60:34451: connect: Connection refused (error 111)
20:37:57.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:57.036798 11138 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:37:57.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:57.037520 11138 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:37:57.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:57.038184 11138 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:37:57.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:57.068771 11138 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:37:57.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:57.069155 11138 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:37:57.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:57.069516 11138 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:37:57.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:57.069844 11138 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:37:57.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:57.070194 11138 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:37:57.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:57.070509 11138 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46757/jwks.json
20:37:57.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:57.105046 11138 master_runner.cc:386] Master server non-default flags:
20:37:57.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:37:57.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.185.20:34965
20:37:57.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:37:57.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:37:57.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-2/data
20:37:57.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-2/wal
20:37:57.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:37:57.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.185.62:46507,127.10.185.61:35973,127.10.185.60:34451
20:37:57.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:37:57.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:37:57.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:37:57.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.185.60:34451
20:37:57.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:37:57.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:37:57.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:37:57.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46757/jwks.json
20:37:57.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:37:57.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:37:57.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-2/data/info.pb
20:37:57.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.185.60
20:37:57.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:37:57.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:37:57.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:37:57.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:37:57.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:37:57.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:37:57.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-2/logs
20:37:57.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:37:57.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:37:57.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:37:57.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:37:57.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:37:57.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:37:57.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:37:57.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 18:34:45 UTC on ccycloud.quasar-elnkgk.root.comops.site
20:37:57.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677568
20:37:57.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:37:57.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:57.106734 11138 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:37:57.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:57.108951 11138 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:37:57.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:57.130831 11149 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:37:57.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:57.140177 11136 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.185.60" port: 34451 } attempt: 4
20:37:57.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:57.143910 11136 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.185.60:34451: Network error: Client connection negotiation failed: client connection to 127.10.185.60:34451: connect: Connection refused (error 111)
20:37:57.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:57.183549 11060 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.185.61" port: 35973 } attempt: 7
20:37:57.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:57.199391 11060 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.185.60" port: 34451 } has no permanent_uuid. Determining permanent_uuid...
20:37:57.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:57.205169 11060 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.185.60:34451: Network error: Client connection negotiation failed: client connection to 127.10.185.60:34451: connect: Connection refused (error 111)
20:37:57.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:57.263768 11060 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.185.60" port: 34451 } attempt: 1
20:37:57.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:57.268951 11060 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.185.60:34451: Network error: Client connection negotiation failed: client connection to 127.10.185.60:34451: connect: Connection refused (error 111)
20:37:57.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:57.350395 11060 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.185.60" port: 34451 } attempt: 2
20:37:57.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:57.354827 11060 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.185.60:34451: Network error: Client connection negotiation failed: client connection to 127.10.185.60:34451: connect: Connection refused (error 111)
20:37:57.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:57.440315 11060 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.185.60" port: 34451 } attempt: 3
20:37:57.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:57.443356 11136 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.185.60" port: 34451 } attempt: 5
20:37:57.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:57.445137 11060 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.185.60:34451: Network error: Client connection negotiation failed: client connection to 127.10.185.60:34451: connect: Connection refused (error 111)
20:37:57.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:57.447947 11136 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.185.60:34451: Network error: Client connection negotiation failed: client connection to 127.10.185.60:34451: connect: Connection refused (error 111)
20:37:57.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:57.585673 11060 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.185.60" port: 34451 } attempt: 4
20:37:57.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:57.590862 11060 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.185.60:34451: Network error: Client connection negotiation failed: client connection to 127.10.185.60:34451: connect: Connection refused (error 111)
20:37:57.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:57.874300 11060 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.185.60" port: 34451 } attempt: 5
20:37:57.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:57.879560 11060 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.185.60:34451: Network error: Client connection negotiation failed: client connection to 127.10.185.60:34451: connect: Connection refused (error 111)
20:37:57.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:57.995365 11136 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.185.60" port: 34451 } attempt: 6
20:37:58.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:58.002627 11136 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.185.60:34451: Network error: Client connection negotiation failed: client connection to 127.10.185.60:34451: connect: Connection refused (error 111)
20:37:58.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:57.131793 11148 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:37:58.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:57.134346 11138 server_base.cc:1034] running on GCE node
20:37:58.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:57.132488 11151 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:37:58.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:58.359968 11138 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:37:58.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:58.363461 11138 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:37:58.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:58.364946 11138 hybrid_clock.cc:648] HybridClock initialized: now 1790627878364911 us; error 55 us; skew 500 ppm
20:37:58.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:58.365779 11138 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:37:58.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:58.373299 11138 webserver.cc:460] Webserver started at http://127.10.185.60:37497/ using document root <none> and password file <none>
20:37:58.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:58.374409 11138 fs_manager.cc:359] Metadata directory not provided
20:37:58.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:58.374612 11138 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:37:58.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:58.375007 11138 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:37:58.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:58.381117 11138 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-2/data/instance:
20:37:58.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c313506f80344bbaadcfa391ed39d0bb"
20:37:58.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 20:37:58 on dist-test-slave-qb38"
20:37:58.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:58.382196 11138 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-2/wal/instance:
20:37:58.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c313506f80344bbaadcfa391ed39d0bb"
20:37:58.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 20:37:58 on dist-test-slave-qb38"
20:37:58.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:58.390748 11138 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.006s sys 0.001s
20:37:58.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:58.396276 11162 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:37:58.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:58.397294 11138 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.001s
20:37:58.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:58.397679 11138 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-2/data,/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-2/wal
20:37:58.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c313506f80344bbaadcfa391ed39d0bb"
20:37:58.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 20:37:58 on dist-test-slave-qb38"
20:37:58.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:58.398039 11138 fs_report.cc:383] FS layout report
20:37:58.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:37:58.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-2/wal
20:37:58.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-2/wal
20:37:58.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-2/data/data
20:37:58.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:37:58.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:37:58.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:37:58.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:37:58.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:37:58.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:37:58.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:37:58.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:37:58.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:37:58.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:37:58.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:37:58.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:58.432013 11060 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.185.60" port: 34451 } attempt: 6
20:37:58.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:58.439601 11060 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.185.60:34451: Network error: Client connection negotiation failed: client connection to 127.10.185.60:34451: connect: Connection refused (error 111)
20:37:58.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:58.440397 11138 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:37:58.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:58.441946 11138 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:37:58.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:58.442495 11138 kserver.cc:163] Server-wide thread pool size limit: 3276
20:37:58.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:58.525960 11138 rpc_server.cc:307] RPC server started. Bound to: 127.10.185.60:34451
20:37:58.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:58.526057 11215 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.185.60:34451 every 8 connection(s)
20:37:58.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:58.528743 11138 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-2/data/info.pb
20:37:58.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:58.535005 11216 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:37:58.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:58.537683 10980 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0rSsIl/build/tsan/bin/kudu as pid 11138
20:37:58.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:58.538092 10980 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/master-2/wal/instance
20:37:58.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:58.550247 11216 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.185.62" port: 46507 } has no permanent_uuid. Determining permanent_uuid...
20:37:58.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:58.586477 11216 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.185.61" port: 35973 } has no permanent_uuid. Determining permanent_uuid...
20:37:58.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:58.599259 11216 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.185.60" port: 34451 } has no permanent_uuid. Determining permanent_uuid...
20:37:58.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:58.617311 11216 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c313506f80344bbaadcfa391ed39d0bb: Bootstrap starting.
20:37:58.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:58.625826 11216 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P c313506f80344bbaadcfa391ed39d0bb: Neither blocks nor log segments found. Creating new log.
20:37:58.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:58.628284 11216 log.cc:824] T 00000000000000000000000000000000 P c313506f80344bbaadcfa391ed39d0bb: Log is configured to *not* fsync() on all Append() calls
20:37:58.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:58.633487 11216 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c313506f80344bbaadcfa391ed39d0bb: No bootstrap required, opened a new log
20:37:58.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:58.657706 11216 raft_consensus.cc:348] T 00000000000000000000000000000000 P c313506f80344bbaadcfa391ed39d0bb [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d641128def46413893222c5b2bd5d23b" member_type: VOTER last_known_addr { host: "127.10.185.62" port: 46507 } } peers { permanent_uuid: "1d39e2e4ef6041af8b602bbd15db4dd6" member_type: VOTER last_known_addr { host: "127.10.185.61" port: 35973 } } peers { permanent_uuid: "c313506f80344bbaadcfa391ed39d0bb" member_type: VOTER last_known_addr { host: "127.10.185.60" port: 34451 } }
20:37:58.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:58.658418 11216 raft_consensus.cc:374] T 00000000000000000000000000000000 P c313506f80344bbaadcfa391ed39d0bb [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:37:58.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:58.658670 11216 raft_consensus.cc:729] T 00000000000000000000000000000000 P c313506f80344bbaadcfa391ed39d0bb [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c313506f80344bbaadcfa391ed39d0bb, State: Initialized, Role: FOLLOWER
20:37:58.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:58.659720 11216 consensus_queue.cc:260] T 00000000000000000000000000000000 P c313506f80344bbaadcfa391ed39d0bb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d641128def46413893222c5b2bd5d23b" member_type: VOTER last_known_addr { host: "127.10.185.62" port: 46507 } } peers { permanent_uuid: "1d39e2e4ef6041af8b602bbd15db4dd6" member_type: VOTER last_known_addr { host: "127.10.185.61" port: 35973 } } peers { permanent_uuid: "c313506f80344bbaadcfa391ed39d0bb" member_type: VOTER last_known_addr { host: "127.10.185.60" port: 34451 } }
20:37:58.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:58.663391 11225 sys_catalog.cc:455] T 00000000000000000000000000000000 P c313506f80344bbaadcfa391ed39d0bb [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d641128def46413893222c5b2bd5d23b" member_type: VOTER last_known_addr { host: "127.10.185.62" port: 46507 } } peers { permanent_uuid: "1d39e2e4ef6041af8b602bbd15db4dd6" member_type: VOTER last_known_addr { host: "127.10.185.61" port: 35973 } } peers { permanent_uuid: "c313506f80344bbaadcfa391ed39d0bb" member_type: VOTER last_known_addr { host: "127.10.185.60" port: 34451 } } }
20:37:58.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:58.664907 11225 sys_catalog.cc:458] T 00000000000000000000000000000000 P c313506f80344bbaadcfa391ed39d0bb [sys.catalog]: This master's current role is: FOLLOWER
20:37:58.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:58.666007 11216 sys_catalog.cc:564] T 00000000000000000000000000000000 P c313506f80344bbaadcfa391ed39d0bb [sys.catalog]: configured and running, proceeding with master startup.
20:37:58.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:58.694651 11236 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c313506f80344bbaadcfa391ed39d0bb: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:37:58.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:58.695405 11236 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:37:58.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:58.699120 11170 tablet.cc:2369] T 00000000000000000000000000000000 P c313506f80344bbaadcfa391ed39d0bb: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:37:58.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:58.688544 11138 master_runner.cc:186] Error getting master registration for 127.10.185.62:46507: OK, instance_id { permanent_uuid: "d641128def46413893222c5b2bd5d23b" instance_seqno: 1790627874786876 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:37:58.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:58.730137 11138 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:37:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:58.891754 11225 raft_consensus.cc:482] T 00000000000000000000000000000000 P c313506f80344bbaadcfa391ed39d0bb [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:37:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:58.892323 11225 raft_consensus.cc:504] T 00000000000000000000000000000000 P c313506f80344bbaadcfa391ed39d0bb [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d641128def46413893222c5b2bd5d23b" member_type: VOTER last_known_addr { host: "127.10.185.62" port: 46507 } } peers { permanent_uuid: "1d39e2e4ef6041af8b602bbd15db4dd6" member_type: VOTER last_known_addr { host: "127.10.185.61" port: 35973 } } peers { permanent_uuid: "c313506f80344bbaadcfa391ed39d0bb" member_type: VOTER last_known_addr { host: "127.10.185.60" port: 34451 } }
20:37:58.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:58.894721 11225 leader_election.cc:288] T 00000000000000000000000000000000 P c313506f80344bbaadcfa391ed39d0bb [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d641128def46413893222c5b2bd5d23b (127.10.185.62:46507), 1d39e2e4ef6041af8b602bbd15db4dd6 (127.10.185.61:35973)
20:37:59.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.063397 11136 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.185.60" port: 34451 } attempt: 7
20:37:59.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.087361 11136 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1d39e2e4ef6041af8b602bbd15db4dd6: Bootstrap starting.
20:37:59.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.095710 11136 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 1d39e2e4ef6041af8b602bbd15db4dd6: Neither blocks nor log segments found. Creating new log.
20:37:59.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.099253 11136 log.cc:824] T 00000000000000000000000000000000 P 1d39e2e4ef6041af8b602bbd15db4dd6: Log is configured to *not* fsync() on all Append() calls
20:37:59.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.106308 11136 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1d39e2e4ef6041af8b602bbd15db4dd6: No bootstrap required, opened a new log
20:37:59.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.133893 11136 raft_consensus.cc:348] T 00000000000000000000000000000000 P 1d39e2e4ef6041af8b602bbd15db4dd6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d641128def46413893222c5b2bd5d23b" member_type: VOTER last_known_addr { host: "127.10.185.62" port: 46507 } } peers { permanent_uuid: "1d39e2e4ef6041af8b602bbd15db4dd6" member_type: VOTER last_known_addr { host: "127.10.185.61" port: 35973 } } peers { permanent_uuid: "c313506f80344bbaadcfa391ed39d0bb" member_type: VOTER last_known_addr { host: "127.10.185.60" port: 34451 } }
20:37:59.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.135291 11136 raft_consensus.cc:374] T 00000000000000000000000000000000 P 1d39e2e4ef6041af8b602bbd15db4dd6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:37:59.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.135879 11136 raft_consensus.cc:729] T 00000000000000000000000000000000 P 1d39e2e4ef6041af8b602bbd15db4dd6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1d39e2e4ef6041af8b602bbd15db4dd6, State: Initialized, Role: FOLLOWER
20:37:59.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.137324 11136 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1d39e2e4ef6041af8b602bbd15db4dd6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d641128def46413893222c5b2bd5d23b" member_type: VOTER last_known_addr { host: "127.10.185.62" port: 46507 } } peers { permanent_uuid: "1d39e2e4ef6041af8b602bbd15db4dd6" member_type: VOTER last_known_addr { host: "127.10.185.61" port: 35973 } } peers { permanent_uuid: "c313506f80344bbaadcfa391ed39d0bb" member_type: VOTER last_known_addr { host: "127.10.185.60" port: 34451 } }
20:37:59.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.142407 11240 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1d39e2e4ef6041af8b602bbd15db4dd6 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d641128def46413893222c5b2bd5d23b" member_type: VOTER last_known_addr { host: "127.10.185.62" port: 46507 } } peers { permanent_uuid: "1d39e2e4ef6041af8b602bbd15db4dd6" member_type: VOTER last_known_addr { host: "127.10.185.61" port: 35973 } } peers { permanent_uuid: "c313506f80344bbaadcfa391ed39d0bb" member_type: VOTER last_known_addr { host: "127.10.185.60" port: 34451 } } }
20:37:59.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.144357 11240 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1d39e2e4ef6041af8b602bbd15db4dd6 [sys.catalog]: This master's current role is: FOLLOWER
20:37:59.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.146049 11136 sys_catalog.cc:564] T 00000000000000000000000000000000 P 1d39e2e4ef6041af8b602bbd15db4dd6 [sys.catalog]: configured and running, proceeding with master startup.
20:37:59.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.146311 11111 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c313506f80344bbaadcfa391ed39d0bb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1d39e2e4ef6041af8b602bbd15db4dd6" is_pre_election: true
20:37:59.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.148097 11111 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 1d39e2e4ef6041af8b602bbd15db4dd6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c313506f80344bbaadcfa391ed39d0bb in term 0.
20:37:59.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.149734 11164 leader_election.cc:302] T 00000000000000000000000000000000 P c313506f80344bbaadcfa391ed39d0bb [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1d39e2e4ef6041af8b602bbd15db4dd6, c313506f80344bbaadcfa391ed39d0bb; no voters:
20:37:59.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.150816 11225 raft_consensus.cc:2781] T 00000000000000000000000000000000 P c313506f80344bbaadcfa391ed39d0bb [term 0 FOLLOWER]: Leader pre-election won for term 1
20:37:59.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.151419 11225 raft_consensus.cc:482] T 00000000000000000000000000000000 P c313506f80344bbaadcfa391ed39d0bb [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:37:59.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.151907 11225 raft_consensus.cc:3037] T 00000000000000000000000000000000 P c313506f80344bbaadcfa391ed39d0bb [term 0 FOLLOWER]: Advancing to term 1
20:37:59.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.161603 11225 raft_consensus.cc:504] T 00000000000000000000000000000000 P c313506f80344bbaadcfa391ed39d0bb [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d641128def46413893222c5b2bd5d23b" member_type: VOTER last_known_addr { host: "127.10.185.62" port: 46507 } } peers { permanent_uuid: "1d39e2e4ef6041af8b602bbd15db4dd6" member_type: VOTER last_known_addr { host: "127.10.185.61" port: 35973 } } peers { permanent_uuid: "c313506f80344bbaadcfa391ed39d0bb" member_type: VOTER last_known_addr { host: "127.10.185.60" port: 34451 } }
20:37:59.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.165395 11225 leader_election.cc:288] T 00000000000000000000000000000000 P c313506f80344bbaadcfa391ed39d0bb [CANDIDATE]: Term 1 election: Requested vote from peers d641128def46413893222c5b2bd5d23b (127.10.185.62:46507), 1d39e2e4ef6041af8b602bbd15db4dd6 (127.10.185.61:35973)
20:37:59.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.166051 11111 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c313506f80344bbaadcfa391ed39d0bb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1d39e2e4ef6041af8b602bbd15db4dd6"
20:37:59.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.167095 11111 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 1d39e2e4ef6041af8b602bbd15db4dd6 [term 0 FOLLOWER]: Advancing to term 1
20:37:59.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.185077 11111 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 1d39e2e4ef6041af8b602bbd15db4dd6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c313506f80344bbaadcfa391ed39d0bb in term 1.
20:37:59.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.187924 11164 leader_election.cc:302] T 00000000000000000000000000000000 P c313506f80344bbaadcfa391ed39d0bb [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1d39e2e4ef6041af8b602bbd15db4dd6, c313506f80344bbaadcfa391ed39d0bb; no voters:
20:37:59.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.189203 11225 raft_consensus.cc:2781] T 00000000000000000000000000000000 P c313506f80344bbaadcfa391ed39d0bb [term 1 FOLLOWER]: Leader election won for term 1
20:37:59.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.193864 11225 raft_consensus.cc:686] T 00000000000000000000000000000000 P c313506f80344bbaadcfa391ed39d0bb [term 1 LEADER]: Becoming Leader. State: Replica: c313506f80344bbaadcfa391ed39d0bb, State: Running, Role: LEADER
20:37:59.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.195096 11225 consensus_queue.cc:237] T 00000000000000000000000000000000 P c313506f80344bbaadcfa391ed39d0bb [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d641128def46413893222c5b2bd5d23b" member_type: VOTER last_known_addr { host: "127.10.185.62" port: 46507 } } peers { permanent_uuid: "1d39e2e4ef6041af8b602bbd15db4dd6" member_type: VOTER last_known_addr { host: "127.10.185.61" port: 35973 } } peers { permanent_uuid: "c313506f80344bbaadcfa391ed39d0bb" member_type: VOTER last_known_addr { host: "127.10.185.60" port: 34451 } }
20:37:59.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.224932 11246 sys_catalog.cc:455] T 00000000000000000000000000000000 P c313506f80344bbaadcfa391ed39d0bb [sys.catalog]: SysCatalogTable state changed. Reason: New leader c313506f80344bbaadcfa391ed39d0bb. Latest consensus state: current_term: 1 leader_uuid: "c313506f80344bbaadcfa391ed39d0bb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d641128def46413893222c5b2bd5d23b" member_type: VOTER last_known_addr { host: "127.10.185.62" port: 46507 } } peers { permanent_uuid: "1d39e2e4ef6041af8b602bbd15db4dd6" member_type: VOTER last_known_addr { host: "127.10.185.61" port: 35973 } } peers { permanent_uuid: "c313506f80344bbaadcfa391ed39d0bb" member_type: VOTER last_known_addr { host: "127.10.185.60" port: 34451 } } }
20:37:59.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.226605 11246 sys_catalog.cc:458] T 00000000000000000000000000000000 P c313506f80344bbaadcfa391ed39d0bb [sys.catalog]: This master's current role is: LEADER
20:37:59.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.233568 11253 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
20:37:59.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.245955 11254 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
20:37:59.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.254550 11254 catalog_manager.cc:1484] Initializing Kudu cluster ID...
20:37:59.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.275272 11111 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 1d39e2e4ef6041af8b602bbd15db4dd6 [term 1 FOLLOWER]: Refusing update from remote peer c313506f80344bbaadcfa391ed39d0bb: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:37:59.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.277783 11246 consensus_queue.cc:1035] T 00000000000000000000000000000000 P c313506f80344bbaadcfa391ed39d0bb [LEADER]: Connected to new peer: Peer: permanent_uuid: "1d39e2e4ef6041af8b602bbd15db4dd6" member_type: VOTER last_known_addr { host: "127.10.185.61" port: 35973 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:37:59.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.238723 11062 master_runner.cc:186] Error getting master registration for 127.10.185.62:46507: OK, instance_id { permanent_uuid: "d641128def46413893222c5b2bd5d23b" instance_seqno: 1790627874786876 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:37:59.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.280527 11062 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:37:59.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.300411 11240 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1d39e2e4ef6041af8b602bbd15db4dd6 [sys.catalog]: SysCatalogTable state changed. Reason: New leader c313506f80344bbaadcfa391ed39d0bb. Latest consensus state: current_term: 1 leader_uuid: "c313506f80344bbaadcfa391ed39d0bb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d641128def46413893222c5b2bd5d23b" member_type: VOTER last_known_addr { host: "127.10.185.62" port: 46507 } } peers { permanent_uuid: "1d39e2e4ef6041af8b602bbd15db4dd6" member_type: VOTER last_known_addr { host: "127.10.185.61" port: 35973 } } peers { permanent_uuid: "c313506f80344bbaadcfa391ed39d0bb" member_type: VOTER last_known_addr { host: "127.10.185.60" port: 34451 } } }
20:37:59.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.301265 11240 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1d39e2e4ef6041af8b602bbd15db4dd6 [sys.catalog]: This master's current role is: FOLLOWER
20:37:59.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.307523 11225 sys_catalog.cc:455] T 00000000000000000000000000000000 P c313506f80344bbaadcfa391ed39d0bb [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "c313506f80344bbaadcfa391ed39d0bb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d641128def46413893222c5b2bd5d23b" member_type: VOTER last_known_addr { host: "127.10.185.62" port: 46507 } } peers { permanent_uuid: "1d39e2e4ef6041af8b602bbd15db4dd6" member_type: VOTER last_known_addr { host: "127.10.185.61" port: 35973 } } peers { permanent_uuid: "c313506f80344bbaadcfa391ed39d0bb" member_type: VOTER last_known_addr { host: "127.10.185.60" port: 34451 } } }
20:37:59.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.308202 11225 sys_catalog.cc:458] T 00000000000000000000000000000000 P c313506f80344bbaadcfa391ed39d0bb [sys.catalog]: This master's current role is: LEADER
20:37:59.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.311354 11254 catalog_manager.cc:1347] Generated new cluster ID: b10458906a6a452283fe722e1bd69f93
20:37:59.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.311621 11254 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
20:37:59.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.311486 11240 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1d39e2e4ef6041af8b602bbd15db4dd6 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "c313506f80344bbaadcfa391ed39d0bb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d641128def46413893222c5b2bd5d23b" member_type: VOTER last_known_addr { host: "127.10.185.62" port: 46507 } } peers { permanent_uuid: "1d39e2e4ef6041af8b602bbd15db4dd6" member_type: VOTER last_known_addr { host: "127.10.185.61" port: 35973 } } peers { permanent_uuid: "c313506f80344bbaadcfa391ed39d0bb" member_type: VOTER last_known_addr { host: "127.10.185.60" port: 34451 } } }
20:37:59.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.312194 11240 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1d39e2e4ef6041af8b602bbd15db4dd6 [sys.catalog]: This master's current role is: FOLLOWER
20:37:59.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.380911 11254 catalog_manager.cc:1370] Generated new certificate authority record
20:37:59.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.384223 11254 catalog_manager.cc:1504] Loading token signing keys...
20:37:59.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.407121 11254 catalog_manager.cc:5868] T 00000000000000000000000000000000 P c313506f80344bbaadcfa391ed39d0bb: Generated new TSK 0
20:37:59.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.408838 11254 catalog_manager.cc:1514] Initializing in-progress tserver states...
20:37:59.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.428785 10980 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0rSsIl/build/tsan/bin/kudu
20:37:59.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0rSsIl/build/tsan/bin/kudu
20:37:59.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-0/wal
20:37:59.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-0/data
20:37:59.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:37:59.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:37:59.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:37:59.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:37:59.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:37:59.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:37:59.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-0/logs
20:37:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-0/data/info.pb
20:37:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:37:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:37:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:37:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:37:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:37:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:37:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:37:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:37:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:37:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:37:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.185.1:0
20:37:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.185.1
20:37:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.185.1
20:37:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:37:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.185.62:46507,127.10.185.61:35973,127.10.185.60:34451
20:37:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.185.20:34965
20:37:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:37:59.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:37:59.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:37:59.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.472965 11060 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.185.60" port: 34451 } attempt: 7
20:37:59.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.498677 11060 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d641128def46413893222c5b2bd5d23b: Bootstrap starting.
20:37:59.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.506397 11060 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d641128def46413893222c5b2bd5d23b: Neither blocks nor log segments found. Creating new log.
20:37:59.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.509155 11060 log.cc:824] T 00000000000000000000000000000000 P d641128def46413893222c5b2bd5d23b: Log is configured to *not* fsync() on all Append() calls
20:37:59.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.515161 11060 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d641128def46413893222c5b2bd5d23b: No bootstrap required, opened a new log
20:37:59.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.548475 11060 raft_consensus.cc:348] T 00000000000000000000000000000000 P d641128def46413893222c5b2bd5d23b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d641128def46413893222c5b2bd5d23b" member_type: VOTER last_known_addr { host: "127.10.185.62" port: 46507 } } peers { permanent_uuid: "1d39e2e4ef6041af8b602bbd15db4dd6" member_type: VOTER last_known_addr { host: "127.10.185.61" port: 35973 } } peers { permanent_uuid: "c313506f80344bbaadcfa391ed39d0bb" member_type: VOTER last_known_addr { host: "127.10.185.60" port: 34451 } }
20:37:59.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.549883 11060 raft_consensus.cc:374] T 00000000000000000000000000000000 P d641128def46413893222c5b2bd5d23b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:37:59.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.550177 11060 raft_consensus.cc:729] T 00000000000000000000000000000000 P d641128def46413893222c5b2bd5d23b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d641128def46413893222c5b2bd5d23b, State: Initialized, Role: FOLLOWER
20:37:59.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.551157 11060 consensus_queue.cc:260] T 00000000000000000000000000000000 P d641128def46413893222c5b2bd5d23b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d641128def46413893222c5b2bd5d23b" member_type: VOTER last_known_addr { host: "127.10.185.62" port: 46507 } } peers { permanent_uuid: "1d39e2e4ef6041af8b602bbd15db4dd6" member_type: VOTER last_known_addr { host: "127.10.185.61" port: 35973 } } peers { permanent_uuid: "c313506f80344bbaadcfa391ed39d0bb" member_type: VOTER last_known_addr { host: "127.10.185.60" port: 34451 } }
20:37:59.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.556591 11060 sys_catalog.cc:564] T 00000000000000000000000000000000 P d641128def46413893222c5b2bd5d23b [sys.catalog]: configured and running, proceeding with master startup.
20:37:59.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.556767 11035 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c313506f80344bbaadcfa391ed39d0bb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d641128def46413893222c5b2bd5d23b" is_pre_election: true
20:37:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.558147 11035 raft_consensus.cc:2454] T 00000000000000000000000000000000 P d641128def46413893222c5b2bd5d23b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c313506f80344bbaadcfa391ed39d0bb in term 0.
20:37:59.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.559240 11034 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c313506f80344bbaadcfa391ed39d0bb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d641128def46413893222c5b2bd5d23b"
20:37:59.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.559916 11034 raft_consensus.cc:3037] T 00000000000000000000000000000000 P d641128def46413893222c5b2bd5d23b [term 0 FOLLOWER]: Advancing to term 1
20:37:59.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.555915 11263 sys_catalog.cc:455] T 00000000000000000000000000000000 P d641128def46413893222c5b2bd5d23b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d641128def46413893222c5b2bd5d23b" member_type: VOTER last_known_addr { host: "127.10.185.62" port: 46507 } } peers { permanent_uuid: "1d39e2e4ef6041af8b602bbd15db4dd6" member_type: VOTER last_known_addr { host: "127.10.185.61" port: 35973 } } peers { permanent_uuid: "c313506f80344bbaadcfa391ed39d0bb" member_type: VOTER last_known_addr { host: "127.10.185.60" port: 34451 } } }
20:37:59.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.573591 11034 raft_consensus.cc:2454] T 00000000000000000000000000000000 P d641128def46413893222c5b2bd5d23b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c313506f80344bbaadcfa391ed39d0bb in term 1.
20:37:59.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.577310 11033 raft_consensus.cc:1261] T 00000000000000000000000000000000 P d641128def46413893222c5b2bd5d23b [term 1 FOLLOWER]: Refusing update from remote peer c313506f80344bbaadcfa391ed39d0bb: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:37:59.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.574400 11263 sys_catalog.cc:458] T 00000000000000000000000000000000 P d641128def46413893222c5b2bd5d23b [sys.catalog]: This master's current role is: FOLLOWER
20:37:59.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.591125 11225 consensus_queue.cc:1035] T 00000000000000000000000000000000 P c313506f80344bbaadcfa391ed39d0bb [LEADER]: Connected to new peer: Peer: permanent_uuid: "d641128def46413893222c5b2bd5d23b" member_type: VOTER last_known_addr { host: "127.10.185.62" port: 46507 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
20:37:59.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.651257 11263 sys_catalog.cc:455] T 00000000000000000000000000000000 P d641128def46413893222c5b2bd5d23b [sys.catalog]: SysCatalogTable state changed. Reason: New leader c313506f80344bbaadcfa391ed39d0bb. Latest consensus state: current_term: 1 leader_uuid: "c313506f80344bbaadcfa391ed39d0bb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d641128def46413893222c5b2bd5d23b" member_type: VOTER last_known_addr { host: "127.10.185.62" port: 46507 } } peers { permanent_uuid: "1d39e2e4ef6041af8b602bbd15db4dd6" member_type: VOTER last_known_addr { host: "127.10.185.61" port: 35973 } } peers { permanent_uuid: "c313506f80344bbaadcfa391ed39d0bb" member_type: VOTER last_known_addr { host: "127.10.185.60" port: 34451 } } }
20:37:59.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.653112 11263 sys_catalog.cc:458] T 00000000000000000000000000000000 P d641128def46413893222c5b2bd5d23b [sys.catalog]: This master's current role is: FOLLOWER
20:37:59.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.654631 11270 sys_catalog.cc:455] T 00000000000000000000000000000000 P d641128def46413893222c5b2bd5d23b [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "c313506f80344bbaadcfa391ed39d0bb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d641128def46413893222c5b2bd5d23b" member_type: VOTER last_known_addr { host: "127.10.185.62" port: 46507 } } peers { permanent_uuid: "1d39e2e4ef6041af8b602bbd15db4dd6" member_type: VOTER last_known_addr { host: "127.10.185.61" port: 35973 } } peers { permanent_uuid: "c313506f80344bbaadcfa391ed39d0bb" member_type: VOTER last_known_addr { host: "127.10.185.60" port: 34451 } } }
20:37:59.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.655856 11270 sys_catalog.cc:458] T 00000000000000000000000000000000 P d641128def46413893222c5b2bd5d23b [sys.catalog]: This master's current role is: FOLLOWER
20:37:59.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.679210 11279 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
20:37:59.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:59.906059 11260 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:37:59.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:59.906692 11260 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:37:59.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:59.907325 11260 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:37:59.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:59.942301 11260 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:37:59.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:37:59.943277 11260 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.185.1
20:37:59.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.981055 11260 tablet_server_runner.cc:78] Tablet server non-default flags:
20:37:59.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:37:59.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.185.20:34965
20:37:59.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:37:59.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:37:59.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-0/data
20:37:59.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-0/wal
20:37:59.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:37:59.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:37:59.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.185.1:0
20:37:59.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:37:59.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:37:59.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:37:59.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-0/data/info.pb
20:37:59.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.185.1
20:37:59.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:37:59.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.185.62:46507,127.10.185.61:35973,127.10.185.60:34451
20:37:59.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:37:59.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:37:59.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:37:59.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:37:59.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:37:59.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.185.1
20:37:59.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-0/logs
20:37:59.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:37:59.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:37:59.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:37:59.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:37:59.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:37:59.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:37:59.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:37:59.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 18:34:45 UTC on ccycloud.quasar-elnkgk.root.comops.site
20:37:59.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677568
20:37:59.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:37:59.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.982733 11260 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:37:59.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:37:59.984797 11260 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:38:00.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:00.013509 11285 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:38:00.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:00.201270 11225 sys_catalog.cc:455] T 00000000000000000000000000000000 P c313506f80344bbaadcfa391ed39d0bb [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "c313506f80344bbaadcfa391ed39d0bb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d641128def46413893222c5b2bd5d23b" member_type: VOTER last_known_addr { host: "127.10.185.62" port: 46507 } } peers { permanent_uuid: "1d39e2e4ef6041af8b602bbd15db4dd6" member_type: VOTER last_known_addr { host: "127.10.185.61" port: 35973 } } peers { permanent_uuid: "c313506f80344bbaadcfa391ed39d0bb" member_type: VOTER last_known_addr { host: "127.10.185.60" port: 34451 } } }
20:38:00.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:00.202769 11225 sys_catalog.cc:458] T 00000000000000000000000000000000 P c313506f80344bbaadcfa391ed39d0bb [sys.catalog]: This master's current role is: LEADER
20:38:00.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:00.265203 11253 catalog_manager.cc:1259] Loaded cluster ID: b10458906a6a452283fe722e1bd69f93
20:38:00.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:00.266186 11253 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 1d39e2e4ef6041af8b602bbd15db4dd6: loading cluster ID for follower catalog manager: success
20:38:00.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:00.291599 11253 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 1d39e2e4ef6041af8b602bbd15db4dd6: acquiring CA information for follower catalog manager: success
20:38:00.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:00.308130 11253 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 1d39e2e4ef6041af8b602bbd15db4dd6: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:38:00.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:00.714282 11279 catalog_manager.cc:1259] Loaded cluster ID: b10458906a6a452283fe722e1bd69f93
20:38:00.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:00.715067 11279 catalog_manager.cc:1552] T 00000000000000000000000000000000 P d641128def46413893222c5b2bd5d23b: loading cluster ID for follower catalog manager: success
20:38:00.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:00.740629 11279 catalog_manager.cc:1574] T 00000000000000000000000000000000 P d641128def46413893222c5b2bd5d23b: acquiring CA information for follower catalog manager: success
20:38:00.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:00.765532 11279 catalog_manager.cc:1602] T 00000000000000000000000000000000 P d641128def46413893222c5b2bd5d23b: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:38:01.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:00.017616 11288 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:38:01.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:00.014804 11286 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:38:01.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:01.915320 11287 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1899 milliseconds
20:38:01.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:01.915434 11260 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:38:01.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:01.917169 11260 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:38:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:01.931583 11260 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:38:01.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:01.933454 11260 hybrid_clock.cc:648] HybridClock initialized: now 1790627881933411 us; error 52 us; skew 500 ppm
20:38:01.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:01.934881 11260 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:38:01.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:01.967666 11260 webserver.cc:460] Webserver started at http://127.10.185.1:38537/ using document root <none> and password file <none>
20:38:01.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:01.969400 11260 fs_manager.cc:359] Metadata directory not provided
20:38:01.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:01.972877 11260 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:38:01.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:01.973868 11260 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:38:01.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:01.981765 11260 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-0/data/instance:
20:38:01.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0a8b08265c424d309d9b308a3f907640"
20:38:01.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 20:38:01 on dist-test-slave-qb38"
20:38:01.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:01.983949 11260 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-0/wal/instance:
20:38:01.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0a8b08265c424d309d9b308a3f907640"
20:38:01.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 20:38:01 on dist-test-slave-qb38"
20:38:02.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:02.000950 11260 fs_manager.cc:691] Time spent creating directory manager: real 0.015s user 0.006s sys 0.007s
20:38:02.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:02.015753 11301 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:38:02.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:02.017549 11260 fs_manager.cc:725] Time spent opening block manager: real 0.012s user 0.004s sys 0.007s
20:38:02.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:02.018172 11260 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-0/data,/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-0/wal
20:38:02.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0a8b08265c424d309d9b308a3f907640"
20:38:02.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 20:38:01 on dist-test-slave-qb38"
20:38:02.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:02.018901 11260 fs_report.cc:383] FS layout report
20:38:02.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:38:02.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-0/wal
20:38:02.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-0/wal
20:38:02.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-0/data/data
20:38:02.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:38:02.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:38:02.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:38:02.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:38:02.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:38:02.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:38:02.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:38:02.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:38:02.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:38:02.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:38:02.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:38:02.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:02.104349 11260 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:38:02.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:02.106797 11260 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:38:02.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:02.107800 11260 kserver.cc:163] Server-wide thread pool size limit: 3276
20:38:02.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:02.117534 11260 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
20:38:02.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:02.130254 11260 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:38:02.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:02.130931 11260 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.001s user 0.001s sys 0.000s
20:38:02.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:02.131476 11260 ts_tablet_manager.cc:610] Registered 0 tablets
20:38:02.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:02.131839 11260 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:38:02.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:02.433362 11260 rpc_server.cc:307] RPC server started. Bound to: 127.10.185.1:45669
20:38:02.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:02.433499 11415 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.185.1:45669 every 8 connection(s)
20:38:02.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:02.437307 11260 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-0/data/info.pb
20:38:02.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:02.447068 10980 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0rSsIl/build/tsan/bin/kudu as pid 11260
20:38:02.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:02.447523 10980 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-0/wal/instance
20:38:02.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:02.471122 10980 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0rSsIl/build/tsan/bin/kudu
20:38:02.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0rSsIl/build/tsan/bin/kudu
20:38:02.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-1/wal
20:38:02.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-1/data
20:38:02.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:38:02.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:38:02.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:38:02.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:38:02.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:38:02.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:38:02.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-1/logs
20:38:02.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-1/data/info.pb
20:38:02.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:38:02.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:38:02.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:38:02.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:38:02.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:38:02.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:38:02.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:38:02.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:38:02.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:38:02.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:38:02.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.185.2:0
20:38:02.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.185.2
20:38:02.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.185.2
20:38:02.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:38:02.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.185.62:46507,127.10.185.61:35973,127.10.185.60:34451
20:38:02.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.185.20:34965
20:38:02.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:38:02.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:38:02.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:38:02.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:02.503062 11056 debug-util.cc:398] Leaking SignalData structure 0x7b08000a1480 after lost signal to thread 10993
20:38:02.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:02.507426 11420 heartbeater.cc:347] Connected to a master server at 127.10.185.62:46507
20:38:02.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:02.507953 11420 heartbeater.cc:464] Registering TS with master...
20:38:02.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:02.509359 11420 heartbeater.cc:511] Master 127.10.185.62:46507 requested a full tablet report, sending...
20:38:02.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:02.511734 11417 heartbeater.cc:347] Connected to a master server at 127.10.185.61:35973
20:38:02.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:02.512135 11417 heartbeater.cc:464] Registering TS with master...
20:38:02.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:02.513230 11417 heartbeater.cc:511] Master 127.10.185.61:35973 requested a full tablet report, sending...
20:38:02.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:02.514828 11025 ts_manager.cc:194] Registered new tserver with Master: 0a8b08265c424d309d9b308a3f907640 (127.10.185.1:45669)
20:38:02.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:02.517297 11101 ts_manager.cc:194] Registered new tserver with Master: 0a8b08265c424d309d9b308a3f907640 (127.10.185.1:45669)
20:38:02.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:02.521795 11416 heartbeater.cc:347] Connected to a master server at 127.10.185.60:34451
20:38:02.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:02.522266 11416 heartbeater.cc:464] Registering TS with master...
20:38:02.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:02.523286 11416 heartbeater.cc:511] Master 127.10.185.60:34451 requested a full tablet report, sending...
20:38:02.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:02.527344 11181 ts_manager.cc:194] Registered new tserver with Master: 0a8b08265c424d309d9b308a3f907640 (127.10.185.1:45669)
20:38:02.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:02.532166 11181 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.185.1:36735
20:38:03.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:03.114681 11425 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:38:03.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:03.115873 11425 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:38:03.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:03.117146 11425 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:38:03.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:03.186906 11425 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:38:03.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:03.188757 11425 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.185.2
20:38:03.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:03.267750 11425 tablet_server_runner.cc:78] Tablet server non-default flags:
20:38:03.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:38:03.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.185.20:34965
20:38:03.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:38:03.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:38:03.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-1/data
20:38:03.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-1/wal
20:38:03.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:38:03.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:38:03.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.185.2:0
20:38:03.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:38:03.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:38:03.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:38:03.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-1/data/info.pb
20:38:03.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.185.2
20:38:03.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:38:03.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.185.62:46507,127.10.185.61:35973,127.10.185.60:34451
20:38:03.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:38:03.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:38:03.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:38:03.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:38:03.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:38:03.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.185.2
20:38:03.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-1/logs
20:38:03.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:38:03.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:38:03.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:38:03.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:38:03.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:38:03.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:38:03.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:38:03.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 18:34:45 UTC on ccycloud.quasar-elnkgk.root.comops.site
20:38:03.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677568
20:38:03.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:38:03.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:03.283651 11425 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:38:03.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:03.286659 11425 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:38:03.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:03.346436 11435 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:38:03.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:03.537074 11416 heartbeater.cc:503] Master 127.10.185.60:34451 was elected leader, sending a full tablet report...
20:38:05.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:03.349623 11434 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:38:05.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:03.359162 11437 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:38:05.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:05.288578 11425 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:38:05.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:05.289520 11436 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1939 milliseconds
20:38:05.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:05.291429 11425 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:38:05.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:05.294777 11425 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:38:05.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:05.296470 11425 hybrid_clock.cc:648] HybridClock initialized: now 1790627885296395 us; error 87 us; skew 500 ppm
20:38:05.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:05.297847 11425 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:38:05.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:05.310707 11425 webserver.cc:460] Webserver started at http://127.10.185.2:33931/ using document root <none> and password file <none>
20:38:05.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:05.312031 11425 fs_manager.cc:359] Metadata directory not provided
20:38:05.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:05.312321 11425 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:38:05.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:05.312865 11425 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:38:05.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:05.320317 11425 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-1/data/instance:
20:38:05.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d85aff694f5b4259a36bcfcc55323083"
20:38:05.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 20:38:05 on dist-test-slave-qb38"
20:38:05.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:05.322293 11425 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-1/wal/instance:
20:38:05.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d85aff694f5b4259a36bcfcc55323083"
20:38:05.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 20:38:05 on dist-test-slave-qb38"
20:38:05.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:05.332805 11425 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.008s sys 0.001s
20:38:05.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:05.341053 11445 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:38:05.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:05.342779 11425 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.004s
20:38:05.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:05.343200 11425 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-1/data,/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-1/wal
20:38:05.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d85aff694f5b4259a36bcfcc55323083"
20:38:05.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 20:38:05 on dist-test-slave-qb38"
20:38:05.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:05.343691 11425 fs_report.cc:383] FS layout report
20:38:05.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:38:05.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-1/wal
20:38:05.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-1/wal
20:38:05.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-1/data/data
20:38:05.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:38:05.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:38:05.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:38:05.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:38:05.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:38:05.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:38:05.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:38:05.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:38:05.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:38:05.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:38:05.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:38:05.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:05.395293 11425 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:38:05.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:05.397410 11425 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:38:05.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:05.397991 11425 kserver.cc:163] Server-wide thread pool size limit: 3276
20:38:05.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:05.401635 11425 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
20:38:05.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:05.408094 11425 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:38:05.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:05.408437 11425 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:38:05.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:05.408752 11425 ts_tablet_manager.cc:610] Registered 0 tablets
20:38:05.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:05.408969 11425 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:38:05.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:05.688995 11425 rpc_server.cc:307] RPC server started. Bound to: 127.10.185.2:43143
20:38:05.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:05.689591 11558 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.185.2:43143 every 8 connection(s)
20:38:05.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:05.693023 11425 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-1/data/info.pb
20:38:05.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:05.703459 10980 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0rSsIl/build/tsan/bin/kudu as pid 11425
20:38:05.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:05.704131 10980 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-1/wal/instance
20:38:05.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:05.727223 10980 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0rSsIl/build/tsan/bin/kudu
20:38:05.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0rSsIl/build/tsan/bin/kudu
20:38:05.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-2/wal
20:38:05.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-2/data
20:38:05.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:38:05.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:38:05.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:38:05.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:38:05.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:38:05.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:38:05.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-2/logs
20:38:05.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-2/data/info.pb
20:38:05.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:38:05.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:38:05.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:38:05.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:38:05.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:38:05.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:38:05.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:38:05.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:38:05.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:38:05.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:38:05.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.185.3:0
20:38:05.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.185.3
20:38:05.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.185.3
20:38:05.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:38:05.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.185.62:46507,127.10.185.61:35973,127.10.185.60:34451
20:38:05.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.185.20:34965
20:38:05.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:38:05.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:38:05.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:38:05.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:05.838435 11564 heartbeater.cc:347] Connected to a master server at 127.10.185.62:46507
20:38:05.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:05.839053 11564 heartbeater.cc:464] Registering TS with master...
20:38:05.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:05.840402 11564 heartbeater.cc:511] Master 127.10.185.62:46507 requested a full tablet report, sending...
20:38:05.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:05.843583 11559 heartbeater.cc:347] Connected to a master server at 127.10.185.60:34451
20:38:05.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:05.843971 11559 heartbeater.cc:464] Registering TS with master...
20:38:05.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:05.844084 11025 ts_manager.cc:194] Registered new tserver with Master: d85aff694f5b4259a36bcfcc55323083 (127.10.185.2:43143)
20:38:05.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:05.845343 11559 heartbeater.cc:511] Master 127.10.185.60:34451 requested a full tablet report, sending...
20:38:05.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:05.846900 11561 heartbeater.cc:347] Connected to a master server at 127.10.185.61:35973
20:38:05.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:05.847225 11561 heartbeater.cc:464] Registering TS with master...
20:38:05.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:05.848086 11561 heartbeater.cc:511] Master 127.10.185.61:35973 requested a full tablet report, sending...
20:38:05.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:05.848776 11181 ts_manager.cc:194] Registered new tserver with Master: d85aff694f5b4259a36bcfcc55323083 (127.10.185.2:43143)
20:38:05.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:05.851071 11181 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.185.2:37633
20:38:05.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:05.851351 11101 ts_manager.cc:194] Registered new tserver with Master: d85aff694f5b4259a36bcfcc55323083 (127.10.185.2:43143)
20:38:06.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:06.166539 11560 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:38:06.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:06.167173 11560 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:38:06.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:06.167824 11560 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:38:06.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:06.200248 11560 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:38:06.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:06.201205 11560 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.185.3
20:38:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:06.239063 11560 tablet_server_runner.cc:78] Tablet server non-default flags:
20:38:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:38:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.185.20:34965
20:38:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:38:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:38:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-2/data
20:38:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-2/wal
20:38:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:38:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:38:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.185.3:0
20:38:06.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:38:06.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:38:06.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:38:06.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-2/data/info.pb
20:38:06.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.185.3
20:38:06.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:38:06.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.185.62:46507,127.10.185.61:35973,127.10.185.60:34451
20:38:06.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:38:06.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:38:06.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:38:06.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:38:06.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:38:06.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.185.3
20:38:06.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-2/logs
20:38:06.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:38:06.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:38:06.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:38:06.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:38:06.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:38:06.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:38:06.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:38:06.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 18:34:45 UTC on ccycloud.quasar-elnkgk.root.comops.site
20:38:06.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677568
20:38:06.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:38:06.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:06.240739 11560 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:38:06.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:06.242697 11560 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:38:06.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:06.268914 11578 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:38:06.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:06.856240 11559 heartbeater.cc:503] Master 127.10.185.60:34451 was elected leader, sending a full tablet report...
20:38:07.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:06.269224 11576 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:38:07.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:06.270709 11560 server_base.cc:1034] running on GCE node
20:38:07.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:06.269215 11575 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:38:07.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:07.533986 11560 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:38:07.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:07.536710 11560 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:38:07.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:07.538132 11560 hybrid_clock.cc:648] HybridClock initialized: now 1790627887538081 us; error 52 us; skew 500 ppm
20:38:07.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:07.539244 11560 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:38:07.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:07.555956 11560 webserver.cc:460] Webserver started at http://127.10.185.3:38991/ using document root <none> and password file <none>
20:38:07.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:07.557495 11560 fs_manager.cc:359] Metadata directory not provided
20:38:07.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:07.557809 11560 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:38:07.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:07.558521 11560 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:38:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:07.564312 11560 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-2/data/instance:
20:38:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "83bb047fd9aa4b67914f6607651662b0"
20:38:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 20:38:07 on dist-test-slave-qb38"
20:38:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:07.565567 11560 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-2/wal/instance:
20:38:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "83bb047fd9aa4b67914f6607651662b0"
20:38:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 20:38:07 on dist-test-slave-qb38"
20:38:07.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:07.575460 11560 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.010s sys 0.001s
20:38:07.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:07.583647 11586 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:38:07.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:07.585062 11560 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.001s
20:38:07.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:07.585446 11560 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-2/data,/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-2/wal
20:38:07.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "83bb047fd9aa4b67914f6607651662b0"
20:38:07.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 20:38:07 on dist-test-slave-qb38"
20:38:07.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:07.585769 11560 fs_report.cc:383] FS layout report
20:38:07.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:38:07.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-2/wal
20:38:07.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-2/wal
20:38:07.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-2/data/data
20:38:07.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:38:07.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:38:07.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:38:07.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:38:07.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:38:07.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:38:07.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:38:07.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:38:07.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:38:07.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:38:07.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:38:07.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:07.637184 11560 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:38:07.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:07.638899 11560 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:38:07.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:07.639365 11560 kserver.cc:163] Server-wide thread pool size limit: 3276
20:38:07.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:07.641952 11560 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
20:38:07.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:07.647835 11560 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:38:07.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:07.648094 11560 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:38:07.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:07.648406 11560 ts_tablet_manager.cc:610] Registered 0 tablets
20:38:07.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:07.648552 11560 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:38:07.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:07.836728 11560 rpc_server.cc:307] RPC server started. Bound to: 127.10.185.3:38881
20:38:07.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:07.836851 11699 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.185.3:38881 every 8 connection(s)
20:38:07.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:07.839476 11560 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-2/data/info.pb
20:38:07.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:07.845058 10980 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0rSsIl/build/tsan/bin/kudu as pid 11560
20:38:07.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:07.845660 10980 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster1363372221377564780/ts-2/wal/instance
20:38:07.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:07.895175 11700 heartbeater.cc:347] Connected to a master server at 127.10.185.60:34451
20:38:07.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:07.895692 11700 heartbeater.cc:464] Registering TS with master...
20:38:07.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:07.898276 11700 heartbeater.cc:511] Master 127.10.185.60:34451 requested a full tablet report, sending...
20:38:07.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:07.899323 11702 heartbeater.cc:347] Connected to a master server at 127.10.185.62:46507
20:38:07.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:07.899652 11702 heartbeater.cc:464] Registering TS with master...
20:38:07.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:07.900537 11702 heartbeater.cc:511] Master 127.10.185.62:46507 requested a full tablet report, sending...
20:38:07.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:07.901939 11181 ts_manager.cc:194] Registered new tserver with Master: 83bb047fd9aa4b67914f6607651662b0 (127.10.185.3:38881)
20:38:07.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:07.902448 11701 heartbeater.cc:347] Connected to a master server at 127.10.185.61:35973
20:38:07.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:07.902824 11701 heartbeater.cc:464] Registering TS with master...
20:38:07.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:07.903478 11025 ts_manager.cc:194] Registered new tserver with Master: 83bb047fd9aa4b67914f6607651662b0 (127.10.185.3:38881)
20:38:07.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:07.903640 11701 heartbeater.cc:511] Master 127.10.185.61:35973 requested a full tablet report, sending...
20:38:07.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:07.904311 11181 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.185.3:52219
20:38:07.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:07.906147 11101 ts_manager.cc:194] Registered new tserver with Master: 83bb047fd9aa4b67914f6607651662b0 (127.10.185.3:38881)
20:38:07.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:07.924204 10980 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
20:38:07.936 [DEBUG - main] (MiniKuduCluster.java:195) Response:
20:38:07.937 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
20:38:07.948 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.10.185.62"
port: 46507
}
bound_http_address {
host: "127.10.185.62"
port: 40357
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.10.185.61"
port: 35973
}
bound_http_address {
host: "127.10.185.61"
port: 46783
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.10.185.60"
port: 34451
}
bound_http_address {
host: "127.10.185.60"
port: 37497
}
}
}
20:38:07.955 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
20:38:07.959 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.10.185.1"
port: 45669
}
bound_http_address {
host: "127.10.185.1"
port: 38537
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.10.185.2"
port: 43143
}
bound_http_address {
host: "127.10.185.2"
port: 33931
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.10.185.3"
port: 38881
}
bound_http_address {
host: "127.10.185.3"
port: 38991
}
}
}
20:38:07.959 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
20:38:08.189 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
20:38:08.345 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task0rSsIl/test-tmp
20:38:08.346 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
20:38:08.346 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task0rSsIl/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
20:38:08.350 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
20:38:08.734 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:38:08.735 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
20:38:08.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:08.738780 11711 mini_oidc.cc:152] Starting JWKS server
20:38:08.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:08.744171 11711 webserver.cc:460] Webserver started at http://127.0.0.1:37903/ using document root <none> and password file <none>
20:38:08.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:08.744849 11711 mini_oidc.cc:189] Starting OIDC Discovery server
20:38:08.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:08.747478 11711 webserver.cc:460] Webserver started at http://127.0.0.1:33557/ using document root <none> and password file <none>
20:38:08.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
20:38:08.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/krb5kdc/principal' for realm 'KRBTEST.COM',
20:38:08.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
20:38:08.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 20:38:08 dist-test-slave-qb38 krb5kdc[11723](info): setting up network...
20:38:08.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
20:38:08.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 20:38:08 dist-test-slave-qb38 krb5kdc[11723](info): set up 2 sockets
20:38:08.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 20:38:08 dist-test-slave-qb38 krb5kdc[11723](info): commencing operation
20:38:08.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
20:38:08.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:08.909940 11700 heartbeater.cc:503] Master 127.10.185.60:34451 was elected leader, sending a full tablet report...
20:38:09.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:09.939383 11711 mini_kdc.cc:121] Time spent starting KDC: real 1.157s user 0.009s sys 0.014s
20:38:09.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
20:38:09.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:38:09.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
20:38:09.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
20:38:10.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:38:10.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
20:38:10.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
20:38:10.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:38:10.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
20:38:10.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:38:10.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/krb5kdc/test-user.keytab.
20:38:10.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/krb5kdc/test-user.keytab.
20:38:10.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 20:38:10 dist-test-slave-qb38 krb5kdc[11723](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790627890, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:38:10.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:38:10.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T20:38:10Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
20:38:10.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T20:38:10Z Disabled control of system clock
20:38:10.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.11.111.254@KRBTEST.COM; defaulting to no policy
20:38:10.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:38:10.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.11.111.254@KRBTEST.COM" created.
20:38:10.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:38:10.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.11.111.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/krb5kdc/kudu_127.11.111.254.keytab.
20:38:10.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.11.111.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/krb5kdc/kudu_127.11.111.254.keytab.
20:38:10.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:10.159943 11711 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0rSsIl/build/tsan/bin/kudu
20:38:10.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0rSsIl/build/tsan/bin/kudu
20:38:10.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-0/wal
20:38:10.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-0/data
20:38:10.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:38:10.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:38:10.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:38:10.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:38:10.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:38:10.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:38:10.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-0/logs
20:38:10.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-0/data/info.pb
20:38:10.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:38:10.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:38:10.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:38:10.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:38:10.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:38:10.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:38:10.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:38:10.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:38:10.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:38:10.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:38:10.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:38:10.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:38:10.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.111.254:44619
20:38:10.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.111.254
20:38:10.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:38:10.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/krb5kdc/kudu_127.11.111.254.keytab
20:38:10.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.111.254
20:38:10.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:38:10.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:38:10.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:38:10.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.111.212:36211
20:38:10.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:38:10.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:38:10.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:38:10.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:38:10.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.111.254:44619,127.11.111.253:38599,127.11.111.252:39681
20:38:10.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:38:10.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37903/jwks.json
20:38:10.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:38:10.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:10.504388 11737 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:38:10.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:10.505215 11737 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:38:10.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:10.505965 11737 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:38:10.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:10.547801 11737 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:38:10.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:10.548045 11737 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:38:10.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:10.548398 11737 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:38:10.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:10.548671 11737 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:38:10.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:10.548863 11737 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:38:10.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:10.549053 11737 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:38:10.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:10.549190 11737 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37903/jwks.json
20:38:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:10.590584 11737 master_runner.cc:386] Master server non-default flags:
20:38:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:38:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.111.212:36211
20:38:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:38:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:38:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-0/data
20:38:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-0/wal
20:38:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:38:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:38:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.111.254:44619,127.11.111.253:38599,127.11.111.252:39681
20:38:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/krb5kdc/kudu_127.11.111.254.keytab
20:38:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.111.254
20:38:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:38:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:38:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:38:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.111.254:44619
20:38:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:38:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:38:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:38:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37903/jwks.json
20:38:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:38:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:38:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:38:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:38:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:38:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-0/data/info.pb
20:38:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.111.254
20:38:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:38:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:38:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:38:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:38:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:38:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:38:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-0/logs
20:38:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:38:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:38:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:38:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:38:10.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:38:10.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:38:10.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:38:10.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 18:34:45 UTC on ccycloud.quasar-elnkgk.root.comops.site
20:38:10.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677568
20:38:10.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:38:10.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:10.591719 11737 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:38:10.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:10.592665 11737 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:38:10.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:10.612936 11746 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:38:11.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:10.612926 11744 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:38:11.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:10.613351 11743 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:38:11.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:10.614843 11737 server_base.cc:1034] running on GCE node
20:38:11.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:11.885212 11737 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:38:11.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:11.888513 11737 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:38:11.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:11.890694 11737 hybrid_clock.cc:648] HybridClock initialized: now 1790627891890621 us; error 100 us; skew 500 ppm
20:38:11.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:11.891534 11737 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:38:11.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 20:38:11 dist-test-slave-qb38 krb5kdc[11723](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790627891, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.111.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:38:11.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:11.898389 11737 init.cc:377] Logged in from keytab as kudu/127.11.111.254@KRBTEST.COM (short username kudu)
20:38:11.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:11.905797 11737 webserver.cc:460] Webserver started at http://127.11.111.254:33073/ using document root <none> and password file <none>
20:38:11.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:11.907028 11737 fs_manager.cc:359] Metadata directory not provided
20:38:11.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:11.907292 11737 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:38:11.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:11.907956 11737 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:38:11.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:11.913239 11737 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-0/data/instance:
20:38:11.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "931950d8e4664ae2ab1f2664bb9c5fc5"
20:38:11.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 20:38:11 on dist-test-slave-qb38"
20:38:11.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:11.914597 11737 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-0/wal/instance:
20:38:11.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "931950d8e4664ae2ab1f2664bb9c5fc5"
20:38:11.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 20:38:11 on dist-test-slave-qb38"
20:38:11.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:11.924543 11737 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.005s sys 0.004s
20:38:11.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:11.930756 11754 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:38:11.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:11.932040 11737 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
20:38:11.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:11.932345 11737 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-0/data,/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-0/wal
20:38:11.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "931950d8e4664ae2ab1f2664bb9c5fc5"
20:38:11.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 20:38:11 on dist-test-slave-qb38"
20:38:11.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:11.932652 11737 fs_report.cc:383] FS layout report
20:38:11.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:38:11.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-0/wal
20:38:11.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-0/wal
20:38:11.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-0/data/data
20:38:11.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:38:11.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:38:11.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:38:11.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:38:11.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:38:11.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:38:11.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:38:11.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:38:11.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:38:11.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:38:11.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:38:11.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:11.976708 11737 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:38:11.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:11.985100 11737 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:38:11.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:11.985706 11737 kserver.cc:163] Server-wide thread pool size limit: 3276
20:38:12.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:12.079985 11737 rpc_server.cc:307] RPC server started. Bound to: 127.11.111.254:44619
20:38:12.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:12.080070 11806 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.11.111.254:44619 every 8 connection(s)
20:38:12.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:12.082926 11737 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-0/data/info.pb
20:38:12.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:12.089931 11807 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:38:12.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:12.094004 11711 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0rSsIl/build/tsan/bin/kudu as pid 11737
20:38:12.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:12.094602 11711 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-0/wal/instance
20:38:12.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:12.108402 11807 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.111.254" port: 44619 } has no permanent_uuid. Determining permanent_uuid...
20:38:12.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.11.111.253@KRBTEST.COM; defaulting to no policy
20:38:12.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:38:12.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.11.111.253@KRBTEST.COM" created.
20:38:12.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 20:38:12 dist-test-slave-qb38 krb5kdc[11723](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790627891, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.111.254@KRBTEST.COM for kudu/127.11.111.254@KRBTEST.COM
20:38:12.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:38:12.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.11.111.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/krb5kdc/kudu_127.11.111.253.keytab.
20:38:12.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.11.111.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/krb5kdc/kudu_127.11.111.253.keytab.
20:38:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:12.148591 11711 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0rSsIl/build/tsan/bin/kudu
20:38:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0rSsIl/build/tsan/bin/kudu
20:38:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-1/wal
20:38:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-1/data
20:38:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:38:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:38:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:38:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:38:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:38:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:38:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-1/logs
20:38:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-1/data/info.pb
20:38:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:38:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:38:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:38:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:38:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:38:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:38:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:38:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:38:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:38:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:38:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:38:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:38:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.111.253:38599
20:38:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.111.253
20:38:12.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:38:12.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/krb5kdc/kudu_127.11.111.253.keytab
20:38:12.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.111.253
20:38:12.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:38:12.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:38:12.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:38:12.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.111.212:36211
20:38:12.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:38:12.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:38:12.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:38:12.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:38:12.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.111.254:44619,127.11.111.253:38599,127.11.111.252:39681
20:38:12.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:38:12.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37903/jwks.json
20:38:12.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:38:12.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:12.157042 11807 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.111.253" port: 38599 } has no permanent_uuid. Determining permanent_uuid...
20:38:12.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:12.160323 11757 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.111.253:38599: connect: Connection refused (error 111)
20:38:12.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:12.165398 11807 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.111.253:38599: Network error: Client connection negotiation failed: client connection to 127.11.111.253:38599: connect: Connection refused (error 111)
20:38:12.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:12.215032 11807 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.111.253" port: 38599 } attempt: 1
20:38:12.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:12.218916 11807 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.111.253:38599: Network error: Client connection negotiation failed: client connection to 127.11.111.253:38599: connect: Connection refused (error 111)
20:38:12.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:12.287295 11807 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.111.253" port: 38599 } attempt: 2
20:38:12.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:12.290999 11807 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.111.253:38599: Network error: Client connection negotiation failed: client connection to 127.11.111.253:38599: connect: Connection refused (error 111)
20:38:12.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:12.382433 11807 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.111.253" port: 38599 } attempt: 3
20:38:12.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:12.386989 11807 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.111.253:38599: Network error: Client connection negotiation failed: client connection to 127.11.111.253:38599: connect: Connection refused (error 111)
20:38:12.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:12.494571 11813 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:38:12.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:12.495283 11813 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:38:12.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:12.495914 11813 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:38:12.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:12.527771 11813 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:38:12.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:12.528005 11813 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:38:12.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:12.528259 11813 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:38:12.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:12.528451 11813 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:38:12.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:12.528646 11813 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:38:12.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:12.528898 11813 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:38:12.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:12.529072 11813 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37903/jwks.json
20:38:12.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:12.530378 11807 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.111.253" port: 38599 } attempt: 4
20:38:12.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:12.535125 11807 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.111.253:38599: Network error: Client connection negotiation failed: client connection to 127.11.111.253:38599: connect: Connection refused (error 111)
20:38:12.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:12.569985 11813 master_runner.cc:386] Master server non-default flags:
20:38:12.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:38:12.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.111.212:36211
20:38:12.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:38:12.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:38:12.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-1/data
20:38:12.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-1/wal
20:38:12.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:38:12.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:38:12.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.111.254:44619,127.11.111.253:38599,127.11.111.252:39681
20:38:12.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/krb5kdc/kudu_127.11.111.253.keytab
20:38:12.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.111.253
20:38:12.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:38:12.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:38:12.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:38:12.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.111.253:38599
20:38:12.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:38:12.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:38:12.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:38:12.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37903/jwks.json
20:38:12.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:38:12.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:38:12.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:38:12.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:38:12.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:38:12.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-1/data/info.pb
20:38:12.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.111.253
20:38:12.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:38:12.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:38:12.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:38:12.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:38:12.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:38:12.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:38:12.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-1/logs
20:38:12.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:38:12.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:38:12.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:38:12.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:38:12.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:38:12.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:38:12.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:38:12.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 18:34:45 UTC on ccycloud.quasar-elnkgk.root.comops.site
20:38:12.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677568
20:38:12.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:38:12.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:12.571534 11813 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:38:12.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:12.572963 11813 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:38:12.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:12.595117 11821 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:38:12.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:12.834561 11807 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.111.253" port: 38599 } attempt: 5
20:38:12.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:12.839861 11807 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.111.253:38599: Network error: Client connection negotiation failed: client connection to 127.11.111.253:38599: connect: Connection refused (error 111)
20:38:13.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:13.387353 11807 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.111.253" port: 38599 } attempt: 6
20:38:13.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:13.393983 11807 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.111.253:38599: Network error: Client connection negotiation failed: client connection to 127.11.111.253:38599: connect: Connection refused (error 111)
20:38:13.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:12.595193 11820 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:38:13.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:12.595896 11823 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:38:13.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:12.599071 11813 server_base.cc:1034] running on GCE node
20:38:13.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:13.868361 11813 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:38:13.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:13.871062 11813 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:38:13.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:13.872443 11813 hybrid_clock.cc:648] HybridClock initialized: now 1790627893872405 us; error 78 us; skew 500 ppm
20:38:13.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:13.873417 11813 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:38:13.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 20:38:13 dist-test-slave-qb38 krb5kdc[11723](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790627893, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.111.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:38:13.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:13.885782 11813 init.cc:377] Logged in from keytab as kudu/127.11.111.253@KRBTEST.COM (short username kudu)
20:38:13.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:13.892051 11813 webserver.cc:460] Webserver started at http://127.11.111.253:36951/ using document root <none> and password file <none>
20:38:13.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:13.893337 11813 fs_manager.cc:359] Metadata directory not provided
20:38:13.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:13.893573 11813 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:38:13.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:13.893976 11813 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:38:13.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:13.899641 11813 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-1/data/instance:
20:38:13.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e4792700c86a45feb7fbb59c840418e3"
20:38:13.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 20:38:13 on dist-test-slave-qb38"
20:38:13.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:13.900943 11813 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-1/wal/instance:
20:38:13.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e4792700c86a45feb7fbb59c840418e3"
20:38:13.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 20:38:13 on dist-test-slave-qb38"
20:38:13.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:13.909034 11813 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
20:38:13.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:13.914820 11837 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:38:13.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:13.915916 11813 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
20:38:13.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:13.916301 11813 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-1/data,/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-1/wal
20:38:13.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e4792700c86a45feb7fbb59c840418e3"
20:38:13.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 20:38:13 on dist-test-slave-qb38"
20:38:13.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:13.916702 11813 fs_report.cc:383] FS layout report
20:38:13.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:38:13.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-1/wal
20:38:13.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-1/wal
20:38:13.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-1/data/data
20:38:13.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:38:13.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:38:13.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:38:13.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:38:13.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:38:13.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:38:13.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:38:13.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:38:13.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:38:13.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:38:13.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:38:13.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:13.956200 11813 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:38:13.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:13.962396 11813 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:38:13.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:13.962909 11813 kserver.cc:163] Server-wide thread pool size limit: 3276
20:38:14.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:14.056308 11813 rpc_server.cc:307] RPC server started. Bound to: 127.11.111.253:38599
20:38:14.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:14.056391 11888 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.11.111.253:38599 every 8 connection(s)
20:38:14.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:14.059149 11813 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-1/data/info.pb
20:38:14.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:14.060153 11711 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0rSsIl/build/tsan/bin/kudu as pid 11813
20:38:14.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:14.060667 11711 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-1/wal/instance
20:38:14.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:14.068135 11889 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:38:14.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.11.111.252@KRBTEST.COM; defaulting to no policy
20:38:14.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:38:14.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.11.111.252@KRBTEST.COM" created.
20:38:14.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:14.087540 11889 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.111.254" port: 44619 } has no permanent_uuid. Determining permanent_uuid...
20:38:14.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:38:14.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.11.111.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/krb5kdc/kudu_127.11.111.252.keytab.
20:38:14.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.11.111.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/krb5kdc/kudu_127.11.111.252.keytab.
20:38:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:14.111929 11711 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0rSsIl/build/tsan/bin/kudu
20:38:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0rSsIl/build/tsan/bin/kudu
20:38:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-2/wal
20:38:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-2/data
20:38:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:38:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:38:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:38:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:38:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:38:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:38:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-2/logs
20:38:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-2/data/info.pb
20:38:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:38:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:38:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:38:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:38:14.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:38:14.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:38:14.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:38:14.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:38:14.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:38:14.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:38:14.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:38:14.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:38:14.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.111.252:39681
20:38:14.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.111.252
20:38:14.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:38:14.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/krb5kdc/kudu_127.11.111.252.keytab
20:38:14.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.111.252
20:38:14.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:38:14.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:38:14.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:38:14.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.111.212:36211
20:38:14.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:38:14.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:38:14.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:38:14.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:38:14.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.111.254:44619,127.11.111.253:38599,127.11.111.252:39681
20:38:14.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:38:14.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37903/jwks.json
20:38:14.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:38:14.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 20:38:14 dist-test-slave-qb38 krb5kdc[11723](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790627893, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.111.253@KRBTEST.COM for kudu/127.11.111.254@KRBTEST.COM
20:38:14.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:14.133512 11889 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.111.253" port: 38599 } has no permanent_uuid. Determining permanent_uuid...
20:38:14.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 20:38:14 dist-test-slave-qb38 krb5kdc[11723](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790627893, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.111.253@KRBTEST.COM for kudu/127.11.111.253@KRBTEST.COM
20:38:14.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:14.166085 11889 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.111.252" port: 39681 } has no permanent_uuid. Determining permanent_uuid...
20:38:14.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:14.169030 11839 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.111.252:39681: connect: Connection refused (error 111)
20:38:14.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:14.174103 11889 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.111.252:39681: Network error: Client connection negotiation failed: client connection to 127.11.111.252:39681: connect: Connection refused (error 111)
20:38:14.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:14.223667 11889 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.111.252" port: 39681 } attempt: 1
20:38:14.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:14.227392 11889 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.111.252:39681: Network error: Client connection negotiation failed: client connection to 127.11.111.252:39681: connect: Connection refused (error 111)
20:38:14.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:14.295914 11889 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.111.252" port: 39681 } attempt: 2
20:38:14.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:14.299852 11889 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.111.252:39681: Network error: Client connection negotiation failed: client connection to 127.11.111.252:39681: connect: Connection refused (error 111)
20:38:14.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:14.391937 11889 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.111.252" port: 39681 } attempt: 3
20:38:14.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:14.395952 11889 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.111.252:39681: Network error: Client connection negotiation failed: client connection to 127.11.111.252:39681: connect: Connection refused (error 111)
20:38:14.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:14.454661 11807 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.111.253" port: 38599 } attempt: 7
20:38:14.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 20:38:14 dist-test-slave-qb38 krb5kdc[11723](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790627891, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.111.254@KRBTEST.COM for kudu/127.11.111.253@KRBTEST.COM
20:38:14.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:14.475353 11895 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:38:14.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:14.476102 11895 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:38:14.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:14.477084 11895 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:38:14.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:14.485102 11807 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.111.252" port: 39681 } has no permanent_uuid. Determining permanent_uuid...
20:38:14.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:14.491266 11807 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.111.252:39681: Network error: Client connection negotiation failed: client connection to 127.11.111.252:39681: connect: Connection refused (error 111)
20:38:14.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:14.511657 11895 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:38:14.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:14.511966 11895 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:38:14.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:14.512358 11895 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:38:14.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:14.512691 11895 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:38:14.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:14.512995 11895 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:38:14.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:14.513301 11895 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:38:14.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:14.513533 11895 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37903/jwks.json
20:38:14.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:14.539383 11889 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.111.252" port: 39681 } attempt: 4
20:38:14.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:14.543481 11889 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.111.252:39681: Network error: Client connection negotiation failed: client connection to 127.11.111.252:39681: connect: Connection refused (error 111)
20:38:14.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:14.549651 11807 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.111.252" port: 39681 } attempt: 1
20:38:14.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:14.553836 11807 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.111.252:39681: Network error: Client connection negotiation failed: client connection to 127.11.111.252:39681: connect: Connection refused (error 111)
20:38:14.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:14.555915 11895 master_runner.cc:386] Master server non-default flags:
20:38:14.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:38:14.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.111.212:36211
20:38:14.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:38:14.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:38:14.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-2/data
20:38:14.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-2/wal
20:38:14.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:38:14.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:38:14.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.111.254:44619,127.11.111.253:38599,127.11.111.252:39681
20:38:14.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/krb5kdc/kudu_127.11.111.252.keytab
20:38:14.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.111.252
20:38:14.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:38:14.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:38:14.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:38:14.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.111.252:39681
20:38:14.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:38:14.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:38:14.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:38:14.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37903/jwks.json
20:38:14.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:38:14.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:38:14.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:38:14.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:38:14.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:38:14.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-2/data/info.pb
20:38:14.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.111.252
20:38:14.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:38:14.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:38:14.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:38:14.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:38:14.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:38:14.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:38:14.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-2/logs
20:38:14.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:38:14.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:38:14.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:38:14.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:38:14.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:38:14.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:38:14.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:38:14.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 18:34:45 UTC on ccycloud.quasar-elnkgk.root.comops.site
20:38:14.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677568
20:38:14.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:38:14.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:14.557416 11895 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:38:14.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:14.558683 11895 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:38:14.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:14.581169 11907 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:38:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:14.635314 11807 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.111.252" port: 39681 } attempt: 2
20:38:14.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:14.639732 11807 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.111.252:39681: Network error: Client connection negotiation failed: client connection to 127.11.111.252:39681: connect: Connection refused (error 111)
20:38:14.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:14.725535 11807 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.111.252" port: 39681 } attempt: 3
20:38:14.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:14.730177 11807 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.111.252:39681: Network error: Client connection negotiation failed: client connection to 127.11.111.252:39681: connect: Connection refused (error 111)
20:38:14.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:14.842922 11889 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.111.252" port: 39681 } attempt: 5
20:38:14.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:14.848114 11889 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.111.252:39681: Network error: Client connection negotiation failed: client connection to 127.11.111.252:39681: connect: Connection refused (error 111)
20:38:14.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:14.870651 11807 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.111.252" port: 39681 } attempt: 4
20:38:14.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:14.875982 11807 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.111.252:39681: Network error: Client connection negotiation failed: client connection to 127.11.111.252:39681: connect: Connection refused (error 111)
20:38:15.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:15.159690 11807 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.111.252" port: 39681 } attempt: 5
20:38:15.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:15.165520 11807 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.111.252:39681: Network error: Client connection negotiation failed: client connection to 127.11.111.252:39681: connect: Connection refused (error 111)
20:38:15.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:15.395567 11889 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.111.252" port: 39681 } attempt: 6
20:38:15.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:15.403676 11889 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.111.252:39681: Network error: Client connection negotiation failed: client connection to 127.11.111.252:39681: connect: Connection refused (error 111)
20:38:15.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:15.717996 11807 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.111.252" port: 39681 } attempt: 6
20:38:15.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:15.726074 11807 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.111.252:39681: Network error: Client connection negotiation failed: client connection to 127.11.111.252:39681: connect: Connection refused (error 111)
20:38:15.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:14.581243 11906 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:38:15.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:14.584345 11895 server_base.cc:1034] running on GCE node
20:38:15.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:14.582284 11909 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:38:15.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:15.915807 11895 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:38:15.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:15.919245 11895 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:38:15.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:15.920697 11895 hybrid_clock.cc:648] HybridClock initialized: now 1790627895920652 us; error 69 us; skew 500 ppm
20:38:15.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:15.921538 11895 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:38:15.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 20:38:15 dist-test-slave-qb38 krb5kdc[11723](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790627895, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.111.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:38:15.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:15.928596 11895 init.cc:377] Logged in from keytab as kudu/127.11.111.252@KRBTEST.COM (short username kudu)
20:38:15.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:15.935487 11895 webserver.cc:460] Webserver started at http://127.11.111.252:46361/ using document root <none> and password file <none>
20:38:15.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:15.936846 11895 fs_manager.cc:359] Metadata directory not provided
20:38:15.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:15.937175 11895 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:38:15.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:15.937704 11895 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:38:15.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:15.943298 11895 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-2/data/instance:
20:38:15.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bd803cb0db204f0680cc3df4d41ff9e2"
20:38:15.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 20:38:15 on dist-test-slave-qb38"
20:38:15.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:15.944635 11895 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-2/wal/instance:
20:38:15.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bd803cb0db204f0680cc3df4d41ff9e2"
20:38:15.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 20:38:15 on dist-test-slave-qb38"
20:38:15.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:15.953656 11895 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.009s sys 0.001s
20:38:15.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:15.960603 11922 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:38:15.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:15.961848 11895 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.004s
20:38:15.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:15.962258 11895 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-2/data,/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-2/wal
20:38:15.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bd803cb0db204f0680cc3df4d41ff9e2"
20:38:15.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 20:38:15 on dist-test-slave-qb38"
20:38:15.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:15.962596 11895 fs_report.cc:383] FS layout report
20:38:15.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:38:15.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-2/wal
20:38:15.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-2/wal
20:38:15.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-2/data/data
20:38:15.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:38:15.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:38:15.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:38:15.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:38:15.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:38:15.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:38:15.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:38:15.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:38:15.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:38:15.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:38:15.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:38:16.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:16.013309 11895 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:38:16.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:16.021922 11895 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:38:16.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:16.022518 11895 kserver.cc:163] Server-wide thread pool size limit: 3276
20:38:16.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:16.125536 11895 rpc_server.cc:307] RPC server started. Bound to: 127.11.111.252:39681
20:38:16.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:16.125633 11973 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.11.111.252:39681 every 8 connection(s)
20:38:16.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:16.128341 11895 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-2/data/info.pb
20:38:16.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:16.128975 11711 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0rSsIl/build/tsan/bin/kudu as pid 11895
20:38:16.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:16.129451 11711 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-2/wal/instance
20:38:16.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:16.137639 11974 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:38:16.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:16.156603 11974 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.111.254" port: 44619 } has no permanent_uuid. Determining permanent_uuid...
20:38:16.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 20:38:16 dist-test-slave-qb38 krb5kdc[11723](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790627895, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.111.252@KRBTEST.COM for kudu/127.11.111.254@KRBTEST.COM
20:38:16.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 20:38:16 dist-test-slave-qb38 krb5kdc[11723](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790627890, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.11.111.252@KRBTEST.COM
20:38:16.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:16.198446 11974 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.111.253" port: 38599 } has no permanent_uuid. Determining permanent_uuid...
20:38:16.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 20:38:16 dist-test-slave-qb38 krb5kdc[11723](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790627895, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.111.252@KRBTEST.COM for kudu/127.11.111.253@KRBTEST.COM
20:38:16.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 20:38:16 dist-test-slave-qb38 krb5kdc[11723](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790627890, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.11.111.253@KRBTEST.COM
20:38:16.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:16.231984 11974 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.111.252" port: 39681 } has no permanent_uuid. Determining permanent_uuid...
20:38:16.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 20:38:16 dist-test-slave-qb38 krb5kdc[11723](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790627895, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.111.252@KRBTEST.COM for kudu/127.11.111.252@KRBTEST.COM
20:38:16.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 20:38:16 dist-test-slave-qb38 krb5kdc[11723](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790627890, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.11.111.254@KRBTEST.COM
20:38:16.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:16.270913 11974 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2: Bootstrap starting.
20:38:16.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:16.278805 11974 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2: Neither blocks nor log segments found. Creating new log.
20:38:16.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:16.281119 11974 log.cc:824] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2: Log is configured to *not* fsync() on all Append() calls
20:38:16.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:16.282915 11711 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
20:38:16.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:16.286669 11974 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2: No bootstrap required, opened a new log
20:38:16.293 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:38:16.294 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
20:38:16.306 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
20:38:16.307 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
20:38:16.310 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.11.111.254"
port: 44619
}
bound_http_address {
host: "127.11.111.254"
port: 33073
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.11.111.253"
port: 38599
}
bound_http_address {
host: "127.11.111.253"
port: 36951
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.11.111.252"
port: 39681
}
bound_http_address {
host: "127.11.111.252"
port: 46361
}
}
}
20:38:16.310 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
20:38:16.311 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:38:16.311 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
20:38:16.313 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
20:38:16.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:16.318843 11974 raft_consensus.cc:348] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "931950d8e4664ae2ab1f2664bb9c5fc5" member_type: VOTER last_known_addr { host: "127.11.111.254" port: 44619 } } peers { permanent_uuid: "e4792700c86a45feb7fbb59c840418e3" member_type: VOTER last_known_addr { host: "127.11.111.253" port: 38599 } } peers { permanent_uuid: "bd803cb0db204f0680cc3df4d41ff9e2" member_type: VOTER last_known_addr { host: "127.11.111.252" port: 39681 } }
20:38:16.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:16.319502 11974 raft_consensus.cc:374] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:38:16.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:16.319705 11974 raft_consensus.cc:729] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: bd803cb0db204f0680cc3df4d41ff9e2, State: Initialized, Role: FOLLOWER
20:38:16.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:16.320500 11974 consensus_queue.cc:260] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "931950d8e4664ae2ab1f2664bb9c5fc5" member_type: VOTER last_known_addr { host: "127.11.111.254" port: 44619 } } peers { permanent_uuid: "e4792700c86a45feb7fbb59c840418e3" member_type: VOTER last_known_addr { host: "127.11.111.253" port: 38599 } } peers { permanent_uuid: "bd803cb0db204f0680cc3df4d41ff9e2" member_type: VOTER last_known_addr { host: "127.11.111.252" port: 39681 } }
20:38:16.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:16.324023 11985 sys_catalog.cc:455] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "931950d8e4664ae2ab1f2664bb9c5fc5" member_type: VOTER last_known_addr { host: "127.11.111.254" port: 44619 } } peers { permanent_uuid: "e4792700c86a45feb7fbb59c840418e3" member_type: VOTER last_known_addr { host: "127.11.111.253" port: 38599 } } peers { permanent_uuid: "bd803cb0db204f0680cc3df4d41ff9e2" member_type: VOTER last_known_addr { host: "127.11.111.252" port: 39681 } } }
20:38:16.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:16.324743 11985 sys_catalog.cc:458] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [sys.catalog]: This master's current role is: FOLLOWER
20:38:16.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:16.326363 11974 sys_catalog.cc:564] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [sys.catalog]: configured and running, proceeding with master startup.
20:38:16.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 20:38:16 dist-test-slave-qb38 krb5kdc[11723](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790627896, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:38:16.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:38:16.357 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:38:16.360 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/krb5kdc/krb5cc
20:38:16.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:16.364087 11997 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:38:16.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:16.364467 11997 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:38:16.366 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:38:16.367 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/krb5kdc/krb5cc
20:38:16.368 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:38:16.372 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
20:38:16.374 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
20:38:16.375 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
20:38:16.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:16.356920 11895 master_runner.cc:186] Error getting master registration for 127.11.111.254:44619: OK, instance_id { permanent_uuid: "931950d8e4664ae2ab1f2664bb9c5fc5" instance_seqno: 1790627892055652 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:38:16.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:16.397254 11895 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:38:16.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 20:38:16 dist-test-slave-qb38 krb5kdc[11723](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790627896, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:38:16.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:38:16.413 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:38:16.413 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/krb5kdc/krb5cc
20:38:16.415 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:38:16.416 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.11.111.253:38599
20:38:16.419 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
20:38:16.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:16.419721 11711 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0rSsIl/build/tsan/bin/kudu with pid 11813
20:38:16.449 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:38:16.449 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.11.111.254:44619
20:38:16.450 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
20:38:16.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:16.450644 11711 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0rSsIl/build/tsan/bin/kudu with pid 11737
20:38:16.480 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:38:16.481 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.11.111.252:39681
20:38:16.481 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
20:38:16.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:16.481892 11711 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0rSsIl/build/tsan/bin/kudu with pid 11895
20:38:16.506 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:38:16.506 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.11.111.253:38599
20:38:16.507 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
20:38:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:16.509793 11711 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0rSsIl/build/tsan/bin/kudu
20:38:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0rSsIl/build/tsan/bin/kudu
20:38:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-1/wal
20:38:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-1/data
20:38:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:38:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:38:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:38:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:38:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:38:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:38:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-1/logs
20:38:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-1/data/info.pb
20:38:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:38:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:38:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:38:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:38:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:38:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:38:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:38:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:38:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:38:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:38:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:38:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:38:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.111.253:38599
20:38:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.111.253
20:38:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36951
20:38:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/krb5kdc/kudu_127.11.111.253.keytab
20:38:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.111.253
20:38:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:38:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:38:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:38:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.111.212:36211
20:38:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:38:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:38:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:38:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:38:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.111.254:44619,127.11.111.253:38599,127.11.111.252:39681
20:38:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:38:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37903/jwks.json
20:38:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:38:16.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:16.853731 12001 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:38:16.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:16.854497 12001 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:38:16.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:16.855244 12001 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:38:16.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:16.892797 12001 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:38:16.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:16.893165 12001 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:38:16.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:16.893566 12001 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:38:16.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:16.893785 12001 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:38:16.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:16.893975 12001 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:38:16.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:16.894155 12001 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:38:16.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:16.894431 12001 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37903/jwks.json
20:38:16.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:16.934931 12001 master_runner.cc:386] Master server non-default flags:
20:38:16.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:38:16.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.111.212:36211
20:38:16.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:38:16.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:38:16.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-1/data
20:38:16.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-1/wal
20:38:16.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:38:16.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:38:16.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.111.254:44619,127.11.111.253:38599,127.11.111.252:39681
20:38:16.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/krb5kdc/kudu_127.11.111.253.keytab
20:38:16.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.111.253
20:38:16.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:38:16.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:38:16.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:38:16.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.111.253:38599
20:38:16.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:38:16.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:38:16.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:38:16.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37903/jwks.json
20:38:16.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:38:16.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:38:16.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:38:16.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:38:16.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:38:16.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-1/data/info.pb
20:38:16.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.111.253
20:38:16.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36951
20:38:16.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:38:16.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:38:16.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:38:16.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:38:16.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:38:16.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-1/logs
20:38:16.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:38:16.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:38:16.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:38:16.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:38:16.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:38:16.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:38:16.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:38:16.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 18:34:45 UTC on ccycloud.quasar-elnkgk.root.comops.site
20:38:16.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677568
20:38:16.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:38:16.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:16.936483 12001 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:38:16.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:16.937906 12001 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:38:16.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:16.959035 12007 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:38:18.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:16.961624 12001 server_base.cc:1034] running on GCE node
20:38:18.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:16.961892 12010 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:38:18.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:16.959102 12008 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:38:18.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:18.211122 12001 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:38:18.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:18.214346 12001 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:38:18.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:18.215816 12001 hybrid_clock.cc:648] HybridClock initialized: now 1790627898215780 us; error 53 us; skew 500 ppm
20:38:18.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:18.216743 12001 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:38:18.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 20:38:18 dist-test-slave-qb38 krb5kdc[11723](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790627898, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.111.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:38:18.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:18.223228 12001 init.cc:377] Logged in from keytab as kudu/127.11.111.253@KRBTEST.COM (short username kudu)
20:38:18.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:18.230067 12001 webserver.cc:460] Webserver started at http://127.11.111.253:36951/ using document root <none> and password file <none>
20:38:18.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:18.231112 12001 fs_manager.cc:359] Metadata directory not provided
20:38:18.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:18.231359 12001 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:38:18.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:18.239529 12001 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.003s sys 0.001s
20:38:18.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:18.244190 12019 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:38:18.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:18.245518 12001 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
20:38:18.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:18.245930 12001 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-1/data,/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-1/wal
20:38:18.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e4792700c86a45feb7fbb59c840418e3"
20:38:18.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 20:38:13 on dist-test-slave-qb38"
20:38:18.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:18.248703 12001 fs_report.cc:383] FS layout report
20:38:18.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:38:18.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-1/wal
20:38:18.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-1/wal
20:38:18.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-1/data/data
20:38:18.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:38:18.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:38:18.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:38:18.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:38:18.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:38:18.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:38:18.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:38:18.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:38:18.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:38:18.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:38:18.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:38:18.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:18.294674 12001 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:38:18.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:18.301518 12001 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:38:18.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:18.302188 12001 kserver.cc:163] Server-wide thread pool size limit: 3276
20:38:18.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:18.385329 12001 rpc_server.cc:307] RPC server started. Bound to: 127.11.111.253:38599
20:38:18.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:18.385489 12070 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.11.111.253:38599 every 8 connection(s)
20:38:18.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:18.389046 12001 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-1/data/info.pb
20:38:18.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:18.394306 11711 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0rSsIl/build/tsan/bin/kudu as pid 12001
20:38:18.395 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:38:18.395 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.11.111.254:44619
20:38:18.395 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
20:38:18.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:18.397053 11711 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0rSsIl/build/tsan/bin/kudu
20:38:18.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0rSsIl/build/tsan/bin/kudu
20:38:18.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-0/wal
20:38:18.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-0/data
20:38:18.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:38:18.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:38:18.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:38:18.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:38:18.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:38:18.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:38:18.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-0/logs
20:38:18.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-0/data/info.pb
20:38:18.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:38:18.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:38:18.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:38:18.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:38:18.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:38:18.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:38:18.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:38:18.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:38:18.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:38:18.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:38:18.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:38:18.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:38:18.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.111.254:44619
20:38:18.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.111.254
20:38:18.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33073
20:38:18.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/krb5kdc/kudu_127.11.111.254.keytab
20:38:18.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.111.254
20:38:18.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:38:18.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:38:18.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:38:18.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.111.212:36211
20:38:18.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:38:18.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:38:18.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:38:18.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:38:18.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.111.254:44619,127.11.111.253:38599,127.11.111.252:39681
20:38:18.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:38:18.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37903/jwks.json
20:38:18.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:38:18.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:18.403496 12071 sys_catalog.cc:263] Verifying existing consensus state
20:38:18.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260928 20:38:18.404451 12071 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
20:38:18.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:18.404975 12001 master.cc:537] Master@127.11.111.253:38599 shutting down...
20:38:18.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:18.468564 12001 master.cc:552] Master@127.11.111.253:38599 shutdown complete.
20:38:18.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
20:38:18.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:18.752174 12073 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:38:18.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:18.752743 12073 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:38:18.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:18.753319 12073 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:38:18.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:18.790617 12073 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:38:18.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:18.790843 12073 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:38:18.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:18.791100 12073 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:38:18.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:18.791290 12073 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:38:18.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:18.791492 12073 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:38:18.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:18.791798 12073 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:38:18.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:18.792042 12073 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37903/jwks.json
20:38:18.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:18.830324 12073 master_runner.cc:386] Master server non-default flags:
20:38:18.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:38:18.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.111.212:36211
20:38:18.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:38:18.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:38:18.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-0/data
20:38:18.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-0/wal
20:38:18.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:38:18.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:38:18.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.111.254:44619,127.11.111.253:38599,127.11.111.252:39681
20:38:18.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/krb5kdc/kudu_127.11.111.254.keytab
20:38:18.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.111.254
20:38:18.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:38:18.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:38:18.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:38:18.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.111.254:44619
20:38:18.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:38:18.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:38:18.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:38:18.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37903/jwks.json
20:38:18.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:38:18.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:38:18.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:38:18.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:38:18.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:38:18.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-0/data/info.pb
20:38:18.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.111.254
20:38:18.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33073
20:38:18.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:38:18.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:38:18.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:38:18.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:38:18.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:38:18.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-0/logs
20:38:18.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:38:18.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:38:18.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:38:18.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:38:18.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:38:18.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:38:18.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:38:18.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 18:34:45 UTC on ccycloud.quasar-elnkgk.root.comops.site
20:38:18.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677568
20:38:18.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:38:18.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:18.831679 12073 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:38:18.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:18.833264 12073 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:38:18.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:18.854830 12081 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:38:20.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:18.857362 12073 server_base.cc:1034] running on GCE node
20:38:20.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:18.854907 12080 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:38:20.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:18.854826 12083 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:38:20.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:20.125370 12073 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:38:20.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:20.128218 12073 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:38:20.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:20.129645 12073 hybrid_clock.cc:648] HybridClock initialized: now 1790627900129593 us; error 58 us; skew 500 ppm
20:38:20.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:20.130659 12073 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:38:20.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 20:38:20 dist-test-slave-qb38 krb5kdc[11723](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790627900, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.111.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:38:20.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:20.137501 12073 init.cc:377] Logged in from keytab as kudu/127.11.111.254@KRBTEST.COM (short username kudu)
20:38:20.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:20.143256 12073 webserver.cc:460] Webserver started at http://127.11.111.254:33073/ using document root <none> and password file <none>
20:38:20.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:20.144375 12073 fs_manager.cc:359] Metadata directory not provided
20:38:20.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:20.144702 12073 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:38:20.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:20.153223 12073 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.005s sys 0.000s
20:38:20.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:20.157914 12093 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:38:20.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:20.159319 12073 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
20:38:20.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:20.159771 12073 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-0/data,/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-0/wal
20:38:20.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "931950d8e4664ae2ab1f2664bb9c5fc5"
20:38:20.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 20:38:11 on dist-test-slave-qb38"
20:38:20.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:20.162225 12073 fs_report.cc:383] FS layout report
20:38:20.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:38:20.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-0/wal
20:38:20.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-0/wal
20:38:20.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-0/data/data
20:38:20.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:38:20.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:38:20.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:38:20.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:38:20.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:38:20.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:38:20.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:38:20.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:38:20.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:38:20.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:38:20.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:38:20.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:20.211505 12073 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:38:20.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:20.217852 12073 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:38:20.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:20.218425 12073 kserver.cc:163] Server-wide thread pool size limit: 3276
20:38:20.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:20.291483 12073 rpc_server.cc:307] RPC server started. Bound to: 127.11.111.254:44619
20:38:20.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:20.291625 12144 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.11.111.254:44619 every 8 connection(s)
20:38:20.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:20.294395 12073 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-0/data/info.pb
20:38:20.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:20.295142 11711 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0rSsIl/build/tsan/bin/kudu as pid 12073
20:38:20.296 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:38:20.296 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.11.111.252:39681
20:38:20.297 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
20:38:20.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:20.299288 11711 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0rSsIl/build/tsan/bin/kudu
20:38:20.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0rSsIl/build/tsan/bin/kudu
20:38:20.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-2/wal
20:38:20.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-2/data
20:38:20.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:38:20.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:38:20.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:38:20.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:38:20.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:38:20.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:38:20.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-2/logs
20:38:20.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-2/data/info.pb
20:38:20.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:38:20.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:38:20.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:38:20.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:38:20.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:38:20.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:38:20.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:38:20.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:38:20.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:38:20.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:38:20.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:38:20.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:38:20.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.111.252:39681
20:38:20.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.111.252
20:38:20.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46361
20:38:20.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/krb5kdc/kudu_127.11.111.252.keytab
20:38:20.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.111.252
20:38:20.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:38:20.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:38:20.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:38:20.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.111.212:36211
20:38:20.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:38:20.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:38:20.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:38:20.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:38:20.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.111.254:44619,127.11.111.253:38599,127.11.111.252:39681
20:38:20.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:38:20.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37903/jwks.json
20:38:20.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:38:20.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:20.307013 12145 sys_catalog.cc:263] Verifying existing consensus state
20:38:20.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260928 20:38:20.307855 12145 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
20:38:20.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:20.308259 12073 master.cc:537] Master@127.11.111.254:44619 shutting down...
20:38:20.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:20.360975 12073 master.cc:552] Master@127.11.111.254:44619 shutdown complete.
20:38:20.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
20:38:20.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:20.637198 12147 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:38:20.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:20.637768 12147 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:38:20.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:20.638600 12147 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:38:20.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:20.671811 12147 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:38:20.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:20.672142 12147 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:38:20.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:20.672588 12147 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:38:20.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:20.672822 12147 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:38:20.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:20.673013 12147 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:38:20.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:20.673189 12147 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:38:20.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:20.673316 12147 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37903/jwks.json
20:38:20.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:20.717900 12147 master_runner.cc:386] Master server non-default flags:
20:38:20.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:38:20.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.111.212:36211
20:38:20.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:38:20.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:38:20.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-2/data
20:38:20.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-2/wal
20:38:20.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:38:20.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:38:20.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.111.254:44619,127.11.111.253:38599,127.11.111.252:39681
20:38:20.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/krb5kdc/kudu_127.11.111.252.keytab
20:38:20.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.111.252
20:38:20.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:38:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:38:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:38:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.111.252:39681
20:38:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:38:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:38:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:38:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37903/jwks.json
20:38:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:38:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:38:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:38:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:38:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:38:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-2/data/info.pb
20:38:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.111.252
20:38:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46361
20:38:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:38:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:38:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:38:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:38:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:38:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-2/logs
20:38:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:38:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:38:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:38:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:38:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:38:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:38:20.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:38:20.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 18:34:45 UTC on ccycloud.quasar-elnkgk.root.comops.site
20:38:20.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677568
20:38:20.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:38:20.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:20.719274 12147 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:38:20.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:20.720352 12147 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:38:20.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:20.739059 12153 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:38:21.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:20.741408 12147 server_base.cc:1034] running on GCE node
20:38:21.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:20.739110 12156 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:38:21.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:20.739454 12154 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:38:21.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:21.990784 12147 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:38:21.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:21.994275 12147 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:38:21.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:21.995776 12147 hybrid_clock.cc:648] HybridClock initialized: now 1790627901995752 us; error 62 us; skew 500 ppm
20:38:21.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:21.996865 12147 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:38:22.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 20:38:21 dist-test-slave-qb38 krb5kdc[11723](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790627901, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.111.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:38:22.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:22.004830 12147 init.cc:377] Logged in from keytab as kudu/127.11.111.252@KRBTEST.COM (short username kudu)
20:38:22.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:22.011034 12147 webserver.cc:460] Webserver started at http://127.11.111.252:46361/ using document root <none> and password file <none>
20:38:22.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:22.012161 12147 fs_manager.cc:359] Metadata directory not provided
20:38:22.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:22.012455 12147 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:38:22.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:22.021315 12147 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.000s sys 0.006s
20:38:22.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:22.026111 12166 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:38:22.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:22.027415 12147 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
20:38:22.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:22.027802 12147 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-2/data,/tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-2/wal
20:38:22.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bd803cb0db204f0680cc3df4d41ff9e2"
20:38:22.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 20:38:15 on dist-test-slave-qb38"
20:38:22.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:22.030159 12147 fs_report.cc:383] FS layout report
20:38:22.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:38:22.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-2/wal
20:38:22.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-2/wal
20:38:22.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-2/data/data
20:38:22.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:38:22.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:38:22.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:38:22.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:38:22.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:38:22.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:38:22.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:38:22.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:38:22.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:38:22.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:38:22.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:38:22.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:22.092602 12147 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:38:22.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:22.099051 12147 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:38:22.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:22.099622 12147 kserver.cc:163] Server-wide thread pool size limit: 3276
20:38:22.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:22.173817 12147 rpc_server.cc:307] RPC server started. Bound to: 127.11.111.252:39681
20:38:22.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:22.173957 12217 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.11.111.252:39681 every 8 connection(s)
20:38:22.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:22.177021 12147 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0rSsIl/test-tmp/mini-kudu-cluster5769118045386870786/master-2/data/info.pb
20:38:22.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:22.180370 11711 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0rSsIl/build/tsan/bin/kudu as pid 12147
20:38:22.181 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:38:22.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:22.188195 12218 sys_catalog.cc:263] Verifying existing consensus state
20:38:22.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:22.193928 12218 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2: Bootstrap starting.
20:38:22.231 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.11.111.253:38599(127.11.111.253:38599)] connecting to peer
20:38:22.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:22.231113 12218 log.cc:824] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2: Log is configured to *not* fsync() on all Append() calls
20:38:22.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:22.236323 12218 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:38:22.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:22.237291 12218 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2: Bootstrap complete.
20:38:22.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:22.272298 12218 raft_consensus.cc:348] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "931950d8e4664ae2ab1f2664bb9c5fc5" member_type: VOTER last_known_addr { host: "127.11.111.254" port: 44619 } } peers { permanent_uuid: "e4792700c86a45feb7fbb59c840418e3" member_type: VOTER last_known_addr { host: "127.11.111.253" port: 38599 } } peers { permanent_uuid: "bd803cb0db204f0680cc3df4d41ff9e2" member_type: VOTER last_known_addr { host: "127.11.111.252" port: 39681 } }
20:38:22.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:22.275899 12218 raft_consensus.cc:374] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:38:22.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:22.276217 12218 raft_consensus.cc:729] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: bd803cb0db204f0680cc3df4d41ff9e2, State: Initialized, Role: FOLLOWER
20:38:22.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:22.277343 12218 consensus_queue.cc:260] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "931950d8e4664ae2ab1f2664bb9c5fc5" member_type: VOTER last_known_addr { host: "127.11.111.254" port: 44619 } } peers { permanent_uuid: "e4792700c86a45feb7fbb59c840418e3" member_type: VOTER last_known_addr { host: "127.11.111.253" port: 38599 } } peers { permanent_uuid: "bd803cb0db204f0680cc3df4d41ff9e2" member_type: VOTER last_known_addr { host: "127.11.111.252" port: 39681 } }
20:38:22.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:22.280731 12222 sys_catalog.cc:455] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "931950d8e4664ae2ab1f2664bb9c5fc5" member_type: VOTER last_known_addr { host: "127.11.111.254" port: 44619 } } peers { permanent_uuid: "e4792700c86a45feb7fbb59c840418e3" member_type: VOTER last_known_addr { host: "127.11.111.253" port: 38599 } } peers { permanent_uuid: "bd803cb0db204f0680cc3df4d41ff9e2" member_type: VOTER last_known_addr { host: "127.11.111.252" port: 39681 } } }
20:38:22.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:22.282979 12218 sys_catalog.cc:564] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [sys.catalog]: configured and running, proceeding with master startup.
20:38:22.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:22.284067 12222 sys_catalog.cc:458] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [sys.catalog]: This master's current role is: FOLLOWER
20:38:22.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:22.303897 12233 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
20:38:22.317 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.11.111.254:44619(127.11.111.254:44619)] connecting to peer
20:38:22.318 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.11.111.252:39681(127.11.111.252:39681)] connecting to peer
20:38:22.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:22.313103 12147 master_runner.cc:186] Error getting master registration for 127.11.111.254:44619: Network error: Client connection negotiation failed: client connection to 127.11.111.254:44619: connect: Connection refused (error 111),
20:38:22.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:22.338373 12147 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:38:22.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:22.355134 12172 tablet.cc:2369] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:38:22.369 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:22.369 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:22.370 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.111.253:38599(127.11.111.253:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:22.370 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.111.254:44619(127.11.111.254:44619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:22.372 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.254:44619: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:22.372 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.253:38599: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:22.374 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.11.111.252:39681(127.11.111.252:39681)] Successfully connected to peer
20:38:22.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:22.408155 12222 raft_consensus.cc:482] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:38:22.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:22.408735 12222 raft_consensus.cc:504] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "931950d8e4664ae2ab1f2664bb9c5fc5" member_type: VOTER last_known_addr { host: "127.11.111.254" port: 44619 } } peers { permanent_uuid: "e4792700c86a45feb7fbb59c840418e3" member_type: VOTER last_known_addr { host: "127.11.111.253" port: 38599 } } peers { permanent_uuid: "bd803cb0db204f0680cc3df4d41ff9e2" member_type: VOTER last_known_addr { host: "127.11.111.252" port: 39681 } }
20:38:22.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:22.412722 12222 leader_election.cc:288] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 931950d8e4664ae2ab1f2664bb9c5fc5 (127.11.111.254:44619), e4792700c86a45feb7fbb59c840418e3 (127.11.111.253:38599)
20:38:22.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:22.416555 12169 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.111.254:44619: connect: Connection refused (error 111)
20:38:22.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:22.422092 12169 leader_election.cc:334] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 931950d8e4664ae2ab1f2664bb9c5fc5 (127.11.111.254:44619): Network error: Client connection negotiation failed: client connection to 127.11.111.254:44619: connect: Connection refused (error 111)
20:38:22.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:22.422761 12169 leader_election.cc:334] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e4792700c86a45feb7fbb59c840418e3 (127.11.111.253:38599): Network error: Client connection negotiation failed: client connection to 127.11.111.253:38599: connect: Connection refused (error 111)
20:38:22.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:22.423135 12169 leader_election.cc:302] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: bd803cb0db204f0680cc3df4d41ff9e2; no voters: 931950d8e4664ae2ab1f2664bb9c5fc5, e4792700c86a45feb7fbb59c840418e3
20:38:22.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:22.424178 12222 raft_consensus.cc:2726] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:38:22.465 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
20:38:22.473 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.11.111.252
20:38:22.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 20:38:22 dist-test-slave-qb38 krb5kdc[11723](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790627896, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.11.111.252@KRBTEST.COM
20:38:22.710 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xaf642565, L:/127.0.0.1:59558 - R:/127.11.111.252:39681] using SASL/GSSAPI
20:38:22.726 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681; will retry
20:38:22.747 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.111.253:38599(127.11.111.253:38599)] connecting to peer
20:38:22.749 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.111.254:44619(127.11.111.254:44619)] connecting to peer
20:38:22.750 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:22.751 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.111.253:38599(127.11.111.253:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:22.755 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.253:38599: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:22.756 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:22.757 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.111.254:44619(127.11.111.254:44619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:22.757 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.254:44619: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:22.763 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681; will retry
20:38:22.765 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.111.253:38599(127.11.111.253:38599)] connecting to peer
20:38:22.768 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.111.254:44619(127.11.111.254:44619)] connecting to peer
20:38:22.768 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:22.768 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.111.253:38599(127.11.111.253:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:22.769 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.253:38599: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:22.772 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:22.772 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.111.254:44619(127.11.111.254:44619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:22.773 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.254:44619: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:22.777 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681; will retry
20:38:22.786 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.111.253:38599(127.11.111.253:38599)] connecting to peer
20:38:22.788 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.111.254:44619(127.11.111.254:44619)] connecting to peer
20:38:22.789 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:22.789 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.111.253:38599(127.11.111.253:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:22.790 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.253:38599: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:22.791 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:22.791 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.111.254:44619(127.11.111.254:44619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:22.792 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.254:44619: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:22.794 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681; will retry
20:38:22.826 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.111.253:38599(127.11.111.253:38599)] connecting to peer
20:38:22.827 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.111.254:44619(127.11.111.254:44619)] connecting to peer
20:38:22.829 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:22.829 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.111.253:38599(127.11.111.253:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:22.830 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.253:38599: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:22.834 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:22.834 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.111.254:44619(127.11.111.254:44619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:22.838 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.254:44619: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:22.839 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681; will retry
20:38:22.866 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.111.253:38599(127.11.111.253:38599)] connecting to peer
20:38:22.868 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.111.254:44619(127.11.111.254:44619)] connecting to peer
20:38:22.870 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:22.870 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.111.253:38599(127.11.111.253:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:22.871 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.253:38599: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:22.871 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:22.872 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.111.254:44619(127.11.111.254:44619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:22.872 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.254:44619: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:22.875 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681; will retry
20:38:22.945 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.111.253:38599(127.11.111.253:38599)] connecting to peer
20:38:22.947 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.111.254:44619(127.11.111.254:44619)] connecting to peer
20:38:22.948 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:22.948 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.111.253:38599(127.11.111.253:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:22.949 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.253:38599: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:22.950 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:22.950 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.111.254:44619(127.11.111.254:44619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:22.951 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.254:44619: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:22.953 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681; will retry
20:38:23.026 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.111.253:38599(127.11.111.253:38599)] connecting to peer
20:38:23.028 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.111.254:44619(127.11.111.254:44619)] connecting to peer
20:38:23.029 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:23.030 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.111.253:38599(127.11.111.253:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:23.030 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.253:38599: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:23.031 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:23.031 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.111.254:44619(127.11.111.254:44619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:23.031 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.254:44619: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:23.034 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681; will retry
20:38:23.106 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.111.253:38599(127.11.111.253:38599)] connecting to peer
20:38:23.108 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.111.254:44619(127.11.111.254:44619)] connecting to peer
20:38:23.108 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:23.108 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.111.253:38599(127.11.111.253:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:23.109 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.253:38599: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:23.112 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:23.112 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.111.254:44619(127.11.111.254:44619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:23.114 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.254:44619: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:23.116 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681; will retry
20:38:23.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:23.315585 12233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:38:23.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:23.315932 12233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:38:23.326 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.111.253:38599(127.11.111.253:38599)] connecting to peer
20:38:23.328 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.111.254:44619(127.11.111.254:44619)] connecting to peer
20:38:23.328 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:23.329 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.111.253:38599(127.11.111.253:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:23.330 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.253:38599: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:23.332 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:23.332 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.111.254:44619(127.11.111.254:44619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:23.333 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.254:44619: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:23.338 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681; will retry
20:38:23.646 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.111.253:38599(127.11.111.253:38599)] connecting to peer
20:38:23.648 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.111.254:44619(127.11.111.254:44619)] connecting to peer
20:38:23.650 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:23.650 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.111.253:38599(127.11.111.253:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:23.650 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.253:38599: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:23.651 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:23.651 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.111.254:44619(127.11.111.254:44619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:23.651 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.254:44619: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:23.653 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681; will retry
20:38:24.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:24.148221 12245 raft_consensus.cc:482] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:38:24.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:24.148654 12245 raft_consensus.cc:504] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "931950d8e4664ae2ab1f2664bb9c5fc5" member_type: VOTER last_known_addr { host: "127.11.111.254" port: 44619 } } peers { permanent_uuid: "e4792700c86a45feb7fbb59c840418e3" member_type: VOTER last_known_addr { host: "127.11.111.253" port: 38599 } } peers { permanent_uuid: "bd803cb0db204f0680cc3df4d41ff9e2" member_type: VOTER last_known_addr { host: "127.11.111.252" port: 39681 } }
20:38:24.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:24.150655 12245 leader_election.cc:288] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 931950d8e4664ae2ab1f2664bb9c5fc5 (127.11.111.254:44619), e4792700c86a45feb7fbb59c840418e3 (127.11.111.253:38599)
20:38:24.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:24.156383 12169 leader_election.cc:334] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 931950d8e4664ae2ab1f2664bb9c5fc5 (127.11.111.254:44619): Network error: Client connection negotiation failed: client connection to 127.11.111.254:44619: connect: Connection refused (error 111)
20:38:24.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:24.157011 12169 leader_election.cc:334] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e4792700c86a45feb7fbb59c840418e3 (127.11.111.253:38599): Network error: Client connection negotiation failed: client connection to 127.11.111.253:38599: connect: Connection refused (error 111)
20:38:24.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:24.157390 12169 leader_election.cc:302] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: bd803cb0db204f0680cc3df4d41ff9e2; no voters: 931950d8e4664ae2ab1f2664bb9c5fc5, e4792700c86a45feb7fbb59c840418e3
20:38:24.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:24.158114 12245 raft_consensus.cc:2726] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:38:24.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:24.319348 12233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:38:24.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:24.319677 12233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:38:24.706 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.111.253:38599(127.11.111.253:38599)] connecting to peer
20:38:24.707 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.111.254:44619(127.11.111.254:44619)] connecting to peer
20:38:24.709 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:24.709 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.111.253:38599(127.11.111.253:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:24.709 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.253:38599: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:24.715 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:24.715 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.111.254:44619(127.11.111.254:44619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:24.715 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.254:44619: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:24.716 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681; will retry
20:38:25.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:25.322885 12233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:38:25.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:25.323195 12233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:38:26.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:26.326431 12233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:38:26.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:26.326738 12233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:38:26.786 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.111.253:38599(127.11.111.253:38599)] connecting to peer
20:38:26.787 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.111.254:44619(127.11.111.254:44619)] connecting to peer
20:38:26.788 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:26.788 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.111.253:38599(127.11.111.253:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:26.789 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.253:38599: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:26.791 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:26.791 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.111.254:44619(127.11.111.254:44619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:26.791 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.254:44619: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:26.794 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681; will retry
20:38:26.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:26.873482 12251 raft_consensus.cc:482] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:38:26.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:26.873970 12251 raft_consensus.cc:504] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "931950d8e4664ae2ab1f2664bb9c5fc5" member_type: VOTER last_known_addr { host: "127.11.111.254" port: 44619 } } peers { permanent_uuid: "e4792700c86a45feb7fbb59c840418e3" member_type: VOTER last_known_addr { host: "127.11.111.253" port: 38599 } } peers { permanent_uuid: "bd803cb0db204f0680cc3df4d41ff9e2" member_type: VOTER last_known_addr { host: "127.11.111.252" port: 39681 } }
20:38:26.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:26.875737 12251 leader_election.cc:288] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 931950d8e4664ae2ab1f2664bb9c5fc5 (127.11.111.254:44619), e4792700c86a45feb7fbb59c840418e3 (127.11.111.253:38599)
20:38:26.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:26.881826 12169 leader_election.cc:334] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 931950d8e4664ae2ab1f2664bb9c5fc5 (127.11.111.254:44619): Network error: Client connection negotiation failed: client connection to 127.11.111.254:44619: connect: Connection refused (error 111)
20:38:26.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:26.882470 12169 leader_election.cc:334] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e4792700c86a45feb7fbb59c840418e3 (127.11.111.253:38599): Network error: Client connection negotiation failed: client connection to 127.11.111.253:38599: connect: Connection refused (error 111)
20:38:26.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:26.882723 12169 leader_election.cc:302] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: bd803cb0db204f0680cc3df4d41ff9e2; no voters: 931950d8e4664ae2ab1f2664bb9c5fc5, e4792700c86a45feb7fbb59c840418e3
20:38:26.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:26.883394 12251 raft_consensus.cc:2726] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:38:27.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:27.187453 11555 debug-util.cc:398] Leaking SignalData structure 0x7b08000b4900 after lost signal to thread 11428
20:38:27.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:27.188490 11555 debug-util.cc:398] Leaking SignalData structure 0x7b08000b95c0 after lost signal to thread 11558
20:38:27.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:27.329759 12233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:38:27.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:27.330029 12233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:38:27.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:27.340710 12147 master_runner.cc:186] Error getting master registration for 127.11.111.254:44619: Network error: Client connection negotiation failed: client connection to 127.11.111.254:44619: connect: Connection refused (error 111),
20:38:27.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:27.341202 12147 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:38:28.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:28.333303 12233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:38:28.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:28.333956 12233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:38:29.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:29.336931 12233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:38:29.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:29.337214 12233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:38:29.406 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.111.253:38599(127.11.111.253:38599)] connecting to peer
20:38:29.407 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.111.254:44619(127.11.111.254:44619)] connecting to peer
20:38:29.408 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:29.408 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.111.253:38599(127.11.111.253:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:29.408 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.253:38599: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:29.410 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:29.410 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.111.254:44619(127.11.111.254:44619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:29.411 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.254:44619: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:29.413 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681; will retry
20:38:29.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:29.992548 12258 raft_consensus.cc:482] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:38:29.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:29.992957 12258 raft_consensus.cc:504] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "931950d8e4664ae2ab1f2664bb9c5fc5" member_type: VOTER last_known_addr { host: "127.11.111.254" port: 44619 } } peers { permanent_uuid: "e4792700c86a45feb7fbb59c840418e3" member_type: VOTER last_known_addr { host: "127.11.111.253" port: 38599 } } peers { permanent_uuid: "bd803cb0db204f0680cc3df4d41ff9e2" member_type: VOTER last_known_addr { host: "127.11.111.252" port: 39681 } }
20:38:29.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:29.994541 12258 leader_election.cc:288] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 931950d8e4664ae2ab1f2664bb9c5fc5 (127.11.111.254:44619), e4792700c86a45feb7fbb59c840418e3 (127.11.111.253:38599)
20:38:30.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:30.002985 12169 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.111.254:44619: connect: Connection refused (error 111) [suppressed 5 similar messages]
20:38:30.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:30.005756 12169 leader_election.cc:334] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 931950d8e4664ae2ab1f2664bb9c5fc5 (127.11.111.254:44619): Network error: Client connection negotiation failed: client connection to 127.11.111.254:44619: connect: Connection refused (error 111)
20:38:30.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:30.006459 12169 leader_election.cc:334] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e4792700c86a45feb7fbb59c840418e3 (127.11.111.253:38599): Network error: Client connection negotiation failed: client connection to 127.11.111.253:38599: connect: Connection refused (error 111)
20:38:30.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:30.006927 12169 leader_election.cc:302] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: bd803cb0db204f0680cc3df4d41ff9e2; no voters: 931950d8e4664ae2ab1f2664bb9c5fc5, e4792700c86a45feb7fbb59c840418e3
20:38:30.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:30.007601 12258 raft_consensus.cc:2726] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:38:30.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:30.340039 12233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:38:30.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:30.340327 12233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:38:31.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:31.343924 12233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:38:31.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:31.344264 12233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:38:32.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 20:38:32 dist-test-slave-qb38 krb5kdc[11723](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790627912, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.111.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:38:32.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:32.009263 12161 init.cc:295] Successfully reacquired a new kerberos TGT
20:38:32.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:32.147917 12263 raft_consensus.cc:482] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:38:32.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:32.148384 12263 raft_consensus.cc:504] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "931950d8e4664ae2ab1f2664bb9c5fc5" member_type: VOTER last_known_addr { host: "127.11.111.254" port: 44619 } } peers { permanent_uuid: "e4792700c86a45feb7fbb59c840418e3" member_type: VOTER last_known_addr { host: "127.11.111.253" port: 38599 } } peers { permanent_uuid: "bd803cb0db204f0680cc3df4d41ff9e2" member_type: VOTER last_known_addr { host: "127.11.111.252" port: 39681 } }
20:38:32.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:32.150406 12263 leader_election.cc:288] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 931950d8e4664ae2ab1f2664bb9c5fc5 (127.11.111.254:44619), e4792700c86a45feb7fbb59c840418e3 (127.11.111.253:38599)
20:38:32.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:32.156157 12169 leader_election.cc:334] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 931950d8e4664ae2ab1f2664bb9c5fc5 (127.11.111.254:44619): Network error: Client connection negotiation failed: client connection to 127.11.111.254:44619: connect: Connection refused (error 111)
20:38:32.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:32.156726 12169 leader_election.cc:334] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e4792700c86a45feb7fbb59c840418e3 (127.11.111.253:38599): Network error: Client connection negotiation failed: client connection to 127.11.111.253:38599: connect: Connection refused (error 111)
20:38:32.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:32.157035 12169 leader_election.cc:302] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: bd803cb0db204f0680cc3df4d41ff9e2; no voters: 931950d8e4664ae2ab1f2664bb9c5fc5, e4792700c86a45feb7fbb59c840418e3
20:38:32.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:32.157685 12263 raft_consensus.cc:2726] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:38:32.245 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.111.253:38599(127.11.111.253:38599)] connecting to peer
20:38:32.247 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.111.254:44619(127.11.111.254:44619)] connecting to peer
20:38:32.248 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:32.248 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.111.253:38599(127.11.111.253:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:32.248 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.253:38599: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:32.250 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:32.250 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.111.254:44619(127.11.111.254:44619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:32.250 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.254:44619: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:32.252 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681; will retry
20:38:32.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:32.343580 12147 master_runner.cc:186] Error getting master registration for 127.11.111.254:44619: Network error: Client connection negotiation failed: client connection to 127.11.111.254:44619: connect: Connection refused (error 111),
20:38:32.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:32.344029 12147 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:38:32.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:32.347189 12233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:38:32.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:32.347416 12233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:38:33.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:33.350528 12233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:38:33.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:33.350769 12233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:38:34.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:34.353370 12233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:38:34.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:34.353618 12233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:38:35.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:35.356462 12233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:38:35.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:35.356722 12233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:38:36.085 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.111.253:38599(127.11.111.253:38599)] connecting to peer
20:38:36.086 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.111.254:44619(127.11.111.254:44619)] connecting to peer
20:38:36.088 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:36.088 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.111.253:38599(127.11.111.253:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:36.088 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.253:38599: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:36.089 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:36.089 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.111.254:44619(127.11.111.254:44619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:36.089 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.254:44619: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:36.091 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681; will retry
20:38:36.265 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.111.253:38599(127.11.111.253:38599)] connecting to peer
20:38:36.267 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.111.254:44619(127.11.111.254:44619)] connecting to peer
20:38:36.269 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:36.269 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.111.253:38599(127.11.111.253:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:36.269 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.253:38599: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:36.269 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:36.270 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.111.254:44619(127.11.111.254:44619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:36.270 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.254:44619: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:36.272 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681; will retry
20:38:36.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:36.359915 12233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:38:36.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:36.360165 12233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:38:37.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:37.347054 12147 master_runner.cc:186] Error getting master registration for 127.11.111.254:44619: Network error: Client connection negotiation failed: client connection to 127.11.111.254:44619: connect: Connection refused (error 111),
20:38:37.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:37.347396 12147 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:38:37.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:37.362962 12233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:38:37.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:37.363215 12233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:38:37.366 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.111.253:38599(127.11.111.253:38599)] connecting to peer
20:38:37.367 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.111.254:44619(127.11.111.254:44619)] connecting to peer
20:38:37.368 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:37.368 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.111.253:38599(127.11.111.253:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:37.369 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.253:38599: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:37.370 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:37.370 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.111.254:44619(127.11.111.254:44619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:37.370 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.254:44619: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:37.373 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681; will retry
20:38:38.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:38.365952 12233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:38:38.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:38.366284 12233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:38:39.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:39.369205 12233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:38:39.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:39.369477 12233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:38:39.626 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.111.253:38599(127.11.111.253:38599)] connecting to peer
20:38:39.628 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.111.254:44619(127.11.111.254:44619)] connecting to peer
20:38:39.628 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:39.629 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.111.253:38599(127.11.111.253:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:39.629 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.253:38599: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:39.630 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:39.630 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.111.254:44619(127.11.111.254:44619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:39.631 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.254:44619: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:39.633 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681; will retry
20:38:40.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:40.372505 12233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:38:40.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:40.372764 12233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:38:40.926 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.111.253:38599(127.11.111.253:38599)] connecting to peer
20:38:40.931 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.111.254:44619(127.11.111.254:44619)] connecting to peer
20:38:40.932 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:40.932 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.111.253:38599(127.11.111.253:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:40.932 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.253:38599: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:40.933 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:40.933 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.111.254:44619(127.11.111.254:44619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:40.933 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.254:44619: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:40.937 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681; will retry
20:38:41.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:41.376101 12233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:38:41.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:41.376449 12233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:38:42.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 20:38:42 dist-test-slave-qb38 krb5kdc[11723](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790627922, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.111.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:38:42.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:42.011699 12161 init.cc:295] Successfully reacquired a new kerberos TGT
20:38:42.146 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.111.253:38599(127.11.111.253:38599)] connecting to peer
20:38:42.146 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.111.254:44619(127.11.111.254:44619)] connecting to peer
20:38:42.147 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:42.147 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.111.253:38599(127.11.111.253:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:42.148 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.253:38599: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:42.149 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:42.149 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.111.254:44619(127.11.111.254:44619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:42.150 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.254:44619: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:42.152 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681; will retry
20:38:42.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:42.350869 12147 master_runner.cc:186] Error getting master registration for 127.11.111.254:44619: Network error: Client connection negotiation failed: client connection to 127.11.111.254:44619: connect: Connection refused (error 111),
20:38:42.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:42.351302 12147 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:38:42.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:42.379423 12233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:38:42.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:42.379686 12233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:38:43.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:43.382491 12233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:38:43.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:43.382736 12233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:38:43.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:43.495148 12283 raft_consensus.cc:482] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:38:43.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:43.495648 12283 raft_consensus.cc:504] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "931950d8e4664ae2ab1f2664bb9c5fc5" member_type: VOTER last_known_addr { host: "127.11.111.254" port: 44619 } } peers { permanent_uuid: "e4792700c86a45feb7fbb59c840418e3" member_type: VOTER last_known_addr { host: "127.11.111.253" port: 38599 } } peers { permanent_uuid: "bd803cb0db204f0680cc3df4d41ff9e2" member_type: VOTER last_known_addr { host: "127.11.111.252" port: 39681 } }
20:38:43.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:43.497287 12283 leader_election.cc:288] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 931950d8e4664ae2ab1f2664bb9c5fc5 (127.11.111.254:44619), e4792700c86a45feb7fbb59c840418e3 (127.11.111.253:38599)
20:38:43.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:43.499985 12169 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.111.254:44619: connect: Connection refused (error 111) [suppressed 3 similar messages]
20:38:43.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:43.503140 12169 leader_election.cc:334] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 931950d8e4664ae2ab1f2664bb9c5fc5 (127.11.111.254:44619): Network error: Client connection negotiation failed: client connection to 127.11.111.254:44619: connect: Connection refused (error 111)
20:38:43.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:43.503777 12169 leader_election.cc:334] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e4792700c86a45feb7fbb59c840418e3 (127.11.111.253:38599): Network error: Client connection negotiation failed: client connection to 127.11.111.253:38599: connect: Connection refused (error 111)
20:38:43.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:43.504038 12169 leader_election.cc:302] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: bd803cb0db204f0680cc3df4d41ff9e2; no voters: 931950d8e4664ae2ab1f2664bb9c5fc5, e4792700c86a45feb7fbb59c840418e3
20:38:43.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:43.504585 12283 raft_consensus.cc:2726] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:38:44.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:44.386101 12233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:38:44.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:44.386475 12233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:38:45.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:45.389402 12233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:38:45.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:45.389652 12233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:38:46.166 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.111.253:38599(127.11.111.253:38599)] connecting to peer
20:38:46.167 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.111.254:44619(127.11.111.254:44619)] connecting to peer
20:38:46.167 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:46.168 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.111.253:38599(127.11.111.253:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:46.168 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:46.168 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.253:38599: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:46.168 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.111.254:44619(127.11.111.254:44619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:46.168 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.254:44619: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:46.172 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681; will retry
20:38:46.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:46.392498 12233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:38:46.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:46.392746 12233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:38:47.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:47.353817 12147 master_runner.cc:186] Error getting master registration for 127.11.111.254:44619: Network error: Client connection negotiation failed: client connection to 127.11.111.254:44619: connect: Connection refused (error 111),
20:38:47.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:47.354184 12147 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:38:47.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:47.395498 12233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:38:47.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:47.395792 12233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:38:47.705 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.111.253:38599(127.11.111.253:38599)] connecting to peer
20:38:47.706 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.111.254:44619(127.11.111.254:44619)] connecting to peer
20:38:47.707 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:47.707 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.111.253:38599(127.11.111.253:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:47.707 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.253:38599: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:47.708 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:47.708 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.111.254:44619(127.11.111.254:44619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:47.708 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.254:44619: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:47.711 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681; will retry
20:38:48.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:48.398479 12233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:38:48.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:48.398713 12233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:38:49.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:49.401734 12233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:38:49.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:49.401981 12233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:38:50.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:50.404719 12233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bd803cb0db204f0680cc3df4d41ff9e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:38:50.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:50.405005 12233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:38:50.866 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.111.253:38599(127.11.111.253:38599)] connecting to peer
20:38:50.867 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.111.254:44619(127.11.111.254:44619)] connecting to peer
20:38:50.867 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:50.868 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.111.253:38599(127.11.111.253:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:50.868 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.253:38599: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599
20:38:50.868 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:50.868 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.111.254:44619(127.11.111.254:44619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:50.869 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.111.254:44619: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
20:38:50.872 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681; will retry
20:38:50.876 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=28688), Traces: [0ms] refreshing cache from master, [33ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [129ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [130ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [183ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [183ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [534ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [539ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [557ms] refreshing cache from master, [559ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [561ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [563ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [563ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [569ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [574ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [575ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [577ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [580ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [581ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [584ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [588ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [590ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [597ms] refreshing cache from master, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [601ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [601ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [603ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [605ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [606ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [637ms] refreshing cache from master, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [641ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [645ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [647ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [651ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [677ms] refreshing cache from master, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [680ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [681ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [682ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [684ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [686ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [687ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [757ms] refreshing cache from master, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [760ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [760ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [762ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [764ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [765ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [837ms] refreshing cache from master, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [839ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [841ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [842ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [843ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [845ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [846ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [917ms] refreshing cache from master, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [919ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [920ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [921ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [924ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [927ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [928ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [1137ms] refreshing cache from master, [1137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [1139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [1142ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [1144ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [1144ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [1149ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [1151ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [1458ms] refreshing cache from master, [1458ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [1459ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [1460ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [1462ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [1463ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [1465ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [1465ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [2517ms] refreshing cache from master, [2517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [2519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [2520ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [2521ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [2526ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [2527ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [2528ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [4597ms] refreshing cache from master, [4597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [4599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [4600ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [4600ms] too many traces: truncated at 100 traces, deferred=Deferred@1353456202(state=PENDING, result=null, callback=(continuation of Deferred@1934361732 after retry RPC after error@972085392) -> (continuation of Deferred@1281384736 after retry RPC after error@2083409068) -> (continuation of Deferred@1876238217 after retry RPC after error@1356758715) -> (continuation of Deferred@527989985 after retry RPC after error@297601166) -> (continuation of Deferred@1448309863 after retry RPC after error@1977344944) -> (continuation of Deferred@689440414 after retry RPC after error@1343783499) -> (continuation of Deferred@497362019 after retry RPC after error@935956996) -> (continuation of Deferred@1577864853 after retry RPC after error@1242727877) -> (continuation of Deferred@2116036312 after retry RPC after error@84066986) -> (continuation of Deferred@969301113 after retry RPC after error@1738857001) -> (continuation of Deferred@416102115 after retry RPC after error@197763266) -> (continuation of Deferred@6750294 after retry RPC after error@1104406497) -> (continuation of Deferred@811117334 after retry RPC after error@1985073775) -> (continuation of Deferred@1309415747 after retry RPC after error@280888204) -> (continuation of Deferred@1699416726 after retry RPC after error@502361850) -> (continuation of Deferred@865708550 after retry RPC after error@1398504858) -> (continuation of Deferred@1853473501 after retry RPC after error@2020206422) -> (continuation of Deferred@115260359 after retry RPC after error@241279275) -> (continuation of Deferred@774825232 after retry RPC after error@424338644) -> (continuation of Deferred@1790534841 after retry RPC after error@1775429692) -> (continuation of Deferred@814585459 after retry RPC after error@2125441266) -> (continuation of Deferred@1595049778 after retry RPC after error@570361828) -> (continuation of Deferred@2099061670 after retry RPC after error@2121727183), errback=(continuation of Deferred@1934361732 after retry RPC after error@972085392) -> (continuation of Deferred@1281384736 after retry RPC after error@2083409068) -> (continuation of Deferred@1876238217 after retry RPC after error@1356758715) -> (continuation of Deferred@527989985 after retry RPC after error@297601166) -> (continuation of Deferred@1448309863 after retry RPC after error@1977344944) -> (continuation of Deferred@689440414 after retry RPC after error@1343783499) -> (continuation of Deferred@497362019 after retry RPC after error@935956996) -> (continuation of Deferred@1577864853 after retry RPC after error@1242727877) -> (continuation of Deferred@2116036312 after retry RPC after error@84066986) -> (continuation of Deferred@969301113 after retry RPC after error@1738857001) -> (continuation of Deferred@416102115 after retry RPC after error@197763266) -> (continuation of Deferred@6750294 after retry RPC after error@1104406497) -> (continuation of Deferred@811117334 after retry RPC after error@1985073775) -> (continuation of Deferred@1309415747 after retry RPC after error@280888204) -> (continuation of Deferred@1699416726 after retry RPC after error@502361850) -> (continuation of Deferred@865708550 after retry RPC after error@1398504858) -> (continuation of Deferred@1853473501 after retry RPC after error@2020206422) -> (continuation of Deferred@115260359 after retry RPC after error@241279275) -> (continuation of Deferred@774825232 after retry RPC after error@424338644) -> (continuation of Deferred@1790534841 after retry RPC after error@1775429692) -> (continuation of Deferred@814585459 after retry RPC after error@2125441266) -> (continuation of Deferred@1595049778 after retry RPC after error@570361828) -> (continuation of Deferred@2099061670 after retry RPC after error@2121727183)))
20:38:50.881 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.11.111.252:39681(127.11.111.252:39681)] disconnecting while in state READY
20:38:50.882 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.11.111.254:44619(127.11.111.254:44619)] disconnecting while in state TERMINATED
20:38:50.882 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.11.111.253:38599(127.11.111.253:38599)] disconnecting while in state TERMINATED
20:38:50.887 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:38:50.890 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:38:50.890 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.11.111.252:39681(127.11.111.252:39681)] handling channelInactive
20:38:50.891 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.111.252:39681(127.11.111.252:39681)] cleaning up while in state READY due to: connection closed
20:38:50.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:50.895196 11711 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0rSsIl/build/tsan/bin/kudu with pid 12147
20:38:52.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T20:38:52Z chronyd exiting
20:38:53.109 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:38:53.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:53.111538 10980 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0rSsIl/build/tsan/bin/kudu with pid 11260
20:38:53.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:53.140054 10980 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0rSsIl/build/tsan/bin/kudu with pid 11425
20:38:53.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:53.168181 10980 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0rSsIl/build/tsan/bin/kudu with pid 11560
20:38:53.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:53.197810 10980 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0rSsIl/build/tsan/bin/kudu with pid 10992
20:38:53.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:53.228631 10980 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0rSsIl/build/tsan/bin/kudu with pid 11062
20:38:53.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:53.258452 11164 connection.cc:535] client connection to 127.10.185.61:35973 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
20:38:53.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 20:38:53.258834 11164 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
20:38:53.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 20:38:53.259893 10980 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0rSsIl/build/tsan/bin/kudu with pid 11138
20:38:54.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T20:38:54Z chronyd exiting
20:38:55.434 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=28688), Traces: [0ms] refreshing cache from master, [33ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [129ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [130ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [183ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [183ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [534ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [539ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [557ms] refreshing cache from master, [559ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [561ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [563ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [563ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [569ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [574ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [575ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [577ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [580ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [581ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [584ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [588ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [590ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [597ms] refreshing cache from master, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [601ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [601ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [603ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [605ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [606ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [637ms] refreshing cache from master, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [641ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [645ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [647ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [651ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [677ms] refreshing cache from master, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [680ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [681ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [682ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [684ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [686ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [687ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [757ms] refreshing cache from master, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [760ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [760ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [762ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [764ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [765ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [837ms] refreshing cache from master, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [839ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [841ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [842ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [843ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [845ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [846ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [917ms] refreshing cache from master, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [919ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [920ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [921ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [924ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [927ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [928ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [1137ms] refreshing cache from master, [1137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [1139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [1142ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [1144ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [1144ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [1149ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [1151ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [1458ms] refreshing cache from master, [1458ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [1459ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [1460ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [1462ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [1463ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [1465ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [1465ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [2517ms] refreshing cache from master, [2517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [2519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [2520ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [2521ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [2526ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [2527ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [2528ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [4597ms] refreshing cache from master, [4597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [4599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [4600ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [4600ms] too many traces: truncated at 100 traces, deferred=Deferred@1353456202(state=PENDING, result=null, callback=(continuation of Deferred@1934361732 after retry RPC after error@972085392) -> (continuation of Deferred@1281384736 after retry RPC after error@2083409068) -> (continuation of Deferred@1876238217 after retry RPC after error@1356758715) -> (continuation of Deferred@527989985 after retry RPC after error@297601166) -> (continuation of Deferred@1448309863 after retry RPC after error@1977344944) -> (continuation of Deferred@689440414 after retry RPC after error@1343783499) -> (continuation of Deferred@497362019 after retry RPC after error@935956996) -> (continuation of Deferred@1577864853 after retry RPC after error@1242727877) -> (continuation of Deferred@2116036312 after retry RPC after error@84066986) -> (continuation of Deferred@969301113 after retry RPC after error@1738857001) -> (continuation of Deferred@416102115 after retry RPC after error@197763266) -> (continuation of Deferred@6750294 after retry RPC after error@1104406497) -> (continuation of Deferred@811117334 after retry RPC after error@1985073775) -> (continuation of Deferred@1309415747 after retry RPC after error@280888204) -> (continuation of Deferred@1699416726 after retry RPC after error@502361850) -> (continuation of Deferred@865708550 after retry RPC after error@1398504858) -> (continuation of Deferred@1853473501 after retry RPC after error@2020206422) -> (continuation of Deferred@115260359 after retry RPC after error@241279275) -> (continuation of Deferred@774825232 after retry RPC after error@424338644) -> (continuation of Deferred@1790534841 after retry RPC after error@1775429692) -> (continuation of Deferred@814585459 after retry RPC after error@2125441266) -> (continuation of Deferred@1595049778 after retry RPC after error@570361828) -> (continuation of Deferred@2099061670 after retry RPC after error@2121727183), errback=(continuation of Deferred@1934361732 after retry RPC after error@972085392) -> (continuation of Deferred@1281384736 after retry RPC after error@2083409068) -> (continuation of Deferred@1876238217 after retry RPC after error@1356758715) -> (continuation of Deferred@527989985 after retry RPC after error@297601166) -> (continuation of Deferred@1448309863 after retry RPC after error@1977344944) -> (continuation of Deferred@689440414 after retry RPC after error@1343783499) -> (continuation of Deferred@497362019 after retry RPC after error@935956996) -> (continuation of Deferred@1577864853 after retry RPC after error@1242727877) -> (continuation of Deferred@2116036312 after retry RPC after error@84066986) -> (continuation of Deferred@969301113 after retry RPC after error@1738857001) -> (continuation of Deferred@416102115 after retry RPC after error@197763266) -> (continuation of Deferred@6750294 after retry RPC after error@1104406497) -> (continuation of Deferred@811117334 after retry RPC after error@1985073775) -> (continuation of Deferred@1309415747 after retry RPC after error@280888204) -> (continuation of Deferred@1699416726 after retry RPC after error@502361850) -> (continuation of Deferred@865708550 after retry RPC after error@1398504858) -> (continuation of Deferred@1853473501 after retry RPC after error@2020206422) -> (continuation of Deferred@115260359 after retry RPC after error@241279275) -> (continuation of Deferred@774825232 after retry RPC after error@424338644) -> (continuation of Deferred@1790534841 after retry RPC after error@1775429692) -> (continuation of Deferred@814585459 after retry RPC after error@2125441266) -> (continuation of Deferred@1595049778 after retry RPC after error@570361828) -> (continuation of Deferred@2099061670 after retry RPC after error@2121727183)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.11.111.254:44619
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=28688), Traces: [0ms] refreshing cache from master, [33ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [129ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [130ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [183ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [183ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [534ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [539ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [557ms] refreshing cache from master, [559ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [561ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [563ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [563ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [569ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [574ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [575ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [577ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [580ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [581ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [584ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [588ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [590ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [597ms] refreshing cache from master, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [601ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [601ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [603ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [605ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [606ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [637ms] refreshing cache from master, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [641ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [645ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [647ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [651ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [677ms] refreshing cache from master, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [680ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [681ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [682ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [684ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [686ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [687ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [757ms] refreshing cache from master, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [760ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [760ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [762ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [764ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [765ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [837ms] refreshing cache from master, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [839ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [841ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [842ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [843ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [845ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [846ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [917ms] refreshing cache from master, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [919ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [920ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [921ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [924ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [927ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [928ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [1137ms] refreshing cache from master, [1137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [1139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [1142ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [1144ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [1144ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [1149ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [1151ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [1458ms] refreshing cache from master, [1458ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [1459ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [1460ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [1462ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [1463ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [1465ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [1465ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [2517ms] refreshing cache from master, [2517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [2519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [2520ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [2521ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [2526ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [2527ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [2528ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [4597ms] refreshing cache from master, [4597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [4599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [4600ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [4600ms] too many traces: truncated at 100 traces, deferred=Deferred@1353456202(state=PENDING, result=null, callback=(continuation of Deferred@1934361732 after retry RPC after error@972085392) -> (continuation of Deferred@1281384736 after retry RPC after error@2083409068) -> (continuation of Deferred@1876238217 after retry RPC after error@1356758715) -> (continuation of Deferred@527989985 after retry RPC after error@297601166) -> (continuation of Deferred@1448309863 after retry RPC after error@1977344944) -> (continuation of Deferred@689440414 after retry RPC after error@1343783499) -> (continuation of Deferred@497362019 after retry RPC after error@935956996) -> (continuation of Deferred@1577864853 after retry RPC after error@1242727877) -> (continuation of Deferred@2116036312 after retry RPC after error@84066986) -> (continuation of Deferred@969301113 after retry RPC after error@1738857001) -> (continuation of Deferred@416102115 after retry RPC after error@197763266) -> (continuation of Deferred@6750294 after retry RPC after error@1104406497) -> (continuation of Deferred@811117334 after retry RPC after error@1985073775) -> (continuation of Deferred@1309415747 after retry RPC after error@280888204) -> (continuation of Deferred@1699416726 after retry RPC after error@502361850) -> (continuation of Deferred@865708550 after retry RPC after error@1398504858) -> (continuation of Deferred@1853473501 after retry RPC after error@2020206422) -> (continuation of Deferred@115260359 after retry RPC after error@241279275) -> (continuation of Deferred@774825232 after retry RPC after error@424338644) -> (continuation of Deferred@1790534841 after retry RPC after error@1775429692) -> (continuation of Deferred@814585459 after retry RPC after error@2125441266) -> (continuation of Deferred@1595049778 after retry RPC after error@570361828) -> (continuation of Deferred@2099061670 after retry RPC after error@2121727183), errback=(continuation of Deferred@1934361732 after retry RPC after error@972085392) -> (continuation of Deferred@1281384736 after retry RPC after error@2083409068) -> (continuation of Deferred@1876238217 after retry RPC after error@1356758715) -> (continuation of Deferred@527989985 after retry RPC after error@297601166) -> (continuation of Deferred@1448309863 after retry RPC after error@1977344944) -> (continuation of Deferred@689440414 after retry RPC after error@1343783499) -> (continuation of Deferred@497362019 after retry RPC after error@935956996) -> (continuation of Deferred@1577864853 after retry RPC after error@1242727877) -> (continuation of Deferred@2116036312 after retry RPC after error@84066986) -> (continuation of Deferred@969301113 after retry RPC after error@1738857001) -> (continuation of Deferred@416102115 after retry RPC after error@197763266) -> (continuation of Deferred@6750294 after retry RPC after error@1104406497) -> (continuation of Deferred@811117334 after retry RPC after error@1985073775) -> (continuation of Deferred@1309415747 after retry RPC after error@280888204) -> (continuation of Deferred@1699416726 after retry RPC after error@502361850) -> (continuation of Deferred@865708550 after retry RPC after error@1398504858) -> (continuation of Deferred@1853473501 after retry RPC after error@2020206422) -> (continuation of Deferred@115260359 after retry RPC after error@241279275) -> (continuation of Deferred@774825232 after retry RPC after error@424338644) -> (continuation of Deferred@1790534841 after retry RPC after error@1775429692) -> (continuation of Deferred@814585459 after retry RPC after error@2125441266) -> (continuation of Deferred@1595049778 after retry RPC after error@570361828) -> (continuation of Deferred@2099061670 after retry RPC after error@2121727183)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.11.111.254:44619
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
20:38:56.372 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
20:38:56.373 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=28688), Traces: [0ms] refreshing cache from master, [33ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [129ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [130ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [183ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [183ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [534ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [539ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [557ms] refreshing cache from master, [559ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [561ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [563ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [563ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [569ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [574ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [575ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [577ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [580ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [581ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [584ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [588ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [590ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [597ms] refreshing cache from master, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [601ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [601ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [603ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [605ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [606ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [637ms] refreshing cache from master, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [641ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [645ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [647ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [651ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [677ms] refreshing cache from master, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [680ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [681ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [682ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [684ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [686ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [687ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [757ms] refreshing cache from master, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [760ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [760ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [762ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [764ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [765ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [837ms] refreshing cache from master, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [839ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [841ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [842ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [843ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [845ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [846ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [917ms] refreshing cache from master, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [919ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [920ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [921ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [924ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [927ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [928ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [1137ms] refreshing cache from master, [1137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [1139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [1142ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [1144ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [1144ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [1149ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [1151ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [1458ms] refreshing cache from master, [1458ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [1459ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [1460ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [1462ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [1463ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [1465ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [1465ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [2517ms] refreshing cache from master, [2517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [2519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [2520ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [2521ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [2526ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [2527ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [2528ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [4597ms] refreshing cache from master, [4597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [4599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [4600ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [4600ms] too many traces: truncated at 100 traces, deferred=Deferred@1353456202(state=PENDING, result=null, callback=(continuation of Deferred@1934361732 after retry RPC after error@972085392) -> (continuation of Deferred@1281384736 after retry RPC after error@2083409068) -> (continuation of Deferred@1876238217 after retry RPC after error@1356758715) -> (continuation of Deferred@527989985 after retry RPC after error@297601166) -> (continuation of Deferred@1448309863 after retry RPC after error@1977344944) -> (continuation of Deferred@689440414 after retry RPC after error@1343783499) -> (continuation of Deferred@497362019 after retry RPC after error@935956996) -> (continuation of Deferred@1577864853 after retry RPC after error@1242727877) -> (continuation of Deferred@2116036312 after retry RPC after error@84066986) -> (continuation of Deferred@969301113 after retry RPC after error@1738857001) -> (continuation of Deferred@416102115 after retry RPC after error@197763266) -> (continuation of Deferred@6750294 after retry RPC after error@1104406497) -> (continuation of Deferred@811117334 after retry RPC after error@1985073775) -> (continuation of Deferred@1309415747 after retry RPC after error@280888204) -> (continuation of Deferred@1699416726 after retry RPC after error@502361850) -> (continuation of Deferred@865708550 after retry RPC after error@1398504858) -> (continuation of Deferred@1853473501 after retry RPC after error@2020206422) -> (continuation of Deferred@115260359 after retry RPC after error@241279275) -> (continuation of Deferred@774825232 after retry RPC after error@424338644) -> (continuation of Deferred@1790534841 after retry RPC after error@1775429692) -> (continuation of Deferred@814585459 after retry RPC after error@2125441266) -> (continuation of Deferred@1595049778 after retry RPC after error@570361828) -> (continuation of Deferred@2099061670 after retry RPC after error@2121727183), errback=(continuation of Deferred@1934361732 after retry RPC after error@972085392) -> (continuation of Deferred@1281384736 after retry RPC after error@2083409068) -> (continuation of Deferred@1876238217 after retry RPC after error@1356758715) -> (continuation of Deferred@527989985 after retry RPC after error@297601166) -> (continuation of Deferred@1448309863 after retry RPC after error@1977344944) -> (continuation of Deferred@689440414 after retry RPC after error@1343783499) -> (continuation of Deferred@497362019 after retry RPC after error@935956996) -> (continuation of Deferred@1577864853 after retry RPC after error@1242727877) -> (continuation of Deferred@2116036312 after retry RPC after error@84066986) -> (continuation of Deferred@969301113 after retry RPC after error@1738857001) -> (continuation of Deferred@416102115 after retry RPC after error@197763266) -> (continuation of Deferred@6750294 after retry RPC after error@1104406497) -> (continuation of Deferred@811117334 after retry RPC after error@1985073775) -> (continuation of Deferred@1309415747 after retry RPC after error@280888204) -> (continuation of Deferred@1699416726 after retry RPC after error@502361850) -> (continuation of Deferred@865708550 after retry RPC after error@1398504858) -> (continuation of Deferred@1853473501 after retry RPC after error@2020206422) -> (continuation of Deferred@115260359 after retry RPC after error@241279275) -> (continuation of Deferred@774825232 after retry RPC after error@424338644) -> (continuation of Deferred@1790534841 after retry RPC after error@1775429692) -> (continuation of Deferred@814585459 after retry RPC after error@2125441266) -> (continuation of Deferred@1595049778 after retry RPC after error@570361828) -> (continuation of Deferred@2099061670 after retry RPC after error@2121727183)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.11.111.254:44619
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=28688), Traces: [0ms] refreshing cache from master, [33ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [129ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [130ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [183ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [183ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [534ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [539ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [557ms] refreshing cache from master, [559ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [561ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [563ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [563ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [569ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [574ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [575ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [577ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [580ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [581ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [584ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [588ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [590ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [597ms] refreshing cache from master, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [601ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [601ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [603ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [605ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [606ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [637ms] refreshing cache from master, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [641ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [645ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [647ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [651ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [677ms] refreshing cache from master, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [680ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [681ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [682ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [684ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [686ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [687ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [757ms] refreshing cache from master, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [760ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [760ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [762ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [764ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [765ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [837ms] refreshing cache from master, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [839ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [841ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [842ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [843ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [845ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [846ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [917ms] refreshing cache from master, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [919ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [920ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [921ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [924ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [927ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [928ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [1137ms] refreshing cache from master, [1137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [1139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [1142ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [1144ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [1144ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [1149ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [1151ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [1458ms] refreshing cache from master, [1458ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [1459ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [1460ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [1462ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [1463ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [1465ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [1465ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [2517ms] refreshing cache from master, [2517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [2519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [2520ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [2521ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.253:38599: Network error: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599, [2526ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.252:39681: OK, [2527ms] Sub RPC ConnectToMaster: received response from server master-127.11.111.254:44619: Network error: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [2528ms] delaying RPC due to: Service unavailable: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619, [4597ms] refreshing cache from master, [4597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.253:38599, [4599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.254:44619, [4600ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.111.252:39681, [4600ms] too many traces: truncated at 100 traces, deferred=Deferred@1353456202(state=PENDING, result=null, callback=(continuation of Deferred@1934361732 after retry RPC after error@972085392) -> (continuation of Deferred@1281384736 after retry RPC after error@2083409068) -> (continuation of Deferred@1876238217 after retry RPC after error@1356758715) -> (continuation of Deferred@527989985 after retry RPC after error@297601166) -> (continuation of Deferred@1448309863 after retry RPC after error@1977344944) -> (continuation of Deferred@689440414 after retry RPC after error@1343783499) -> (continuation of Deferred@497362019 after retry RPC after error@935956996) -> (continuation of Deferred@1577864853 after retry RPC after error@1242727877) -> (continuation of Deferred@2116036312 after retry RPC after error@84066986) -> (continuation of Deferred@969301113 after retry RPC after error@1738857001) -> (continuation of Deferred@416102115 after retry RPC after error@197763266) -> (continuation of Deferred@6750294 after retry RPC after error@1104406497) -> (continuation of Deferred@811117334 after retry RPC after error@1985073775) -> (continuation of Deferred@1309415747 after retry RPC after error@280888204) -> (continuation of Deferred@1699416726 after retry RPC after error@502361850) -> (continuation of Deferred@865708550 after retry RPC after error@1398504858) -> (continuation of Deferred@1853473501 after retry RPC after error@2020206422) -> (continuation of Deferred@115260359 after retry RPC after error@241279275) -> (continuation of Deferred@774825232 after retry RPC after error@424338644) -> (continuation of Deferred@1790534841 after retry RPC after error@1775429692) -> (continuation of Deferred@814585459 after retry RPC after error@2125441266) -> (continuation of Deferred@1595049778 after retry RPC after error@570361828) -> (continuation of Deferred@2099061670 after retry RPC after error@2121727183), errback=(continuation of Deferred@1934361732 after retry RPC after error@972085392) -> (continuation of Deferred@1281384736 after retry RPC after error@2083409068) -> (continuation of Deferred@1876238217 after retry RPC after error@1356758715) -> (continuation of Deferred@527989985 after retry RPC after error@297601166) -> (continuation of Deferred@1448309863 after retry RPC after error@1977344944) -> (continuation of Deferred@689440414 after retry RPC after error@1343783499) -> (continuation of Deferred@497362019 after retry RPC after error@935956996) -> (continuation of Deferred@1577864853 after retry RPC after error@1242727877) -> (continuation of Deferred@2116036312 after retry RPC after error@84066986) -> (continuation of Deferred@969301113 after retry RPC after error@1738857001) -> (continuation of Deferred@416102115 after retry RPC after error@197763266) -> (continuation of Deferred@6750294 after retry RPC after error@1104406497) -> (continuation of Deferred@811117334 after retry RPC after error@1985073775) -> (continuation of Deferred@1309415747 after retry RPC after error@280888204) -> (continuation of Deferred@1699416726 after retry RPC after error@502361850) -> (continuation of Deferred@865708550 after retry RPC after error@1398504858) -> (continuation of Deferred@1853473501 after retry RPC after error@2020206422) -> (continuation of Deferred@115260359 after retry RPC after error@241279275) -> (continuation of Deferred@774825232 after retry RPC after error@424338644) -> (continuation of Deferred@1790534841 after retry RPC after error@1775429692) -> (continuation of Deferred@814585459 after retry RPC after error@2125441266) -> (continuation of Deferred@1595049778 after retry RPC after error@570361828) -> (continuation of Deferred@2099061670 after retry RPC after error@2121727183)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.11.111.253:38599,127.11.111.254:44619,127.11.111.252:39681) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.253:38599(127.11.111.253:38599): Connection refused: /127.11.111.253:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.111.254:44619(127.11.111.254:44619): Connection refused: /127.11.111.254:44619
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.11.111.254:44619
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)