07:37:41.462 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
07:37:41.474 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskDxb_lc/test-tmp
07:37:41.476 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
07:37:41.480 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskDxb_lc/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
07:37:41.698 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
07:37:41.885 [DEBUG - main] (MiniKuduCluster.java:195) Response:
07:37:41.886 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
07:37:41.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:41.891836 4163 mini_oidc.cc:152] Starting JWKS server
07:37:41.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:41.896801 4163 webserver.cc:460] Webserver started at http://127.0.0.1:34027/ using document root <none> and password file <none>
07:37:41.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:41.897414 4163 mini_oidc.cc:189] Starting OIDC Discovery server
07:37:41.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:41.899617 4163 webserver.cc:460] Webserver started at http://127.0.0.1:34697/ using document root <none> and password file <none>
07:37:41.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-24T07:37:41Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
07:37:41.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-24T07:37:41Z Disabled control of system clock
07:37:41.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:41.949453 4163 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskDxb_lc/build/tsan/bin/kudu
07:37:41.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDxb_lc/build/tsan/bin/kudu
07:37:41.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-0/wal
07:37:41.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-0/data
07:37:41.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:37:41.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:37:41.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:37:41.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:37:41.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:37:41.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:37:41.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-0/logs
07:37:41.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-0/data/info.pb
07:37:41.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:37:41.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:37:41.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:37:41.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:37:41.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:37:41.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:37:41.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:37:41.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:37:41.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:37:41.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:37:41.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:37:41.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:37:41.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.16.254:41099
07:37:41.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.16.254
07:37:41.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:37:41.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.16.212:39261
07:37:41.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:37:41.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:37:41.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:37:41.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:37:41.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.16.254:41099,127.4.16.253:45839,127.4.16.252:43219
07:37:41.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:37:41.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34027/jwks.json with env {}
07:37:42.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:42.256916 4175 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:37:42.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:42.257472 4175 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:37:42.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:42.258169 4175 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:37:42.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:42.294940 4175 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:37:42.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:42.295248 4175 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:37:42.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:42.295467 4175 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:37:42.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:42.295653 4175 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:37:42.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:42.295908 4175 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:37:42.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:42.296106 4175 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34027/jwks.json
07:37:42.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:42.331163 4175 master_runner.cc:386] Master server non-default flags:
07:37:42.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:37:42.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.16.212:39261
07:37:42.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:37:42.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:37:42.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-0/data
07:37:42.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-0/wal
07:37:42.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:37:42.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.16.254:41099,127.4.16.253:45839,127.4.16.252:43219
07:37:42.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:37:42.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:37:42.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:37:42.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.16.254:41099
07:37:42.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:37:42.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:37:42.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:37:42.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34027/jwks.json
07:37:42.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:37:42.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:37:42.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-0/data/info.pb
07:37:42.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.16.254
07:37:42.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:37:42.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:37:42.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:37:42.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:37:42.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:37:42.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:37:42.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-0/logs
07:37:42.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:37:42.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:37:42.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:37:42.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:37:42.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:37:42.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:37:42.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:37:42.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 06:42:32 UTC on ccycloud.quasar-lgarko.root.comops.site
07:37:42.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669406
07:37:42.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:37:42.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:42.332473 4175 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:37:42.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:42.334263 4175 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:37:42.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:42.354466 4184 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:37:43.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:42.354660 4182 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:37:43.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:42.355410 4175 server_base.cc:1034] running on GCE node
07:37:43.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:42.356117 4181 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:37:43.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:43.543015 4175 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:37:43.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:43.545696 4175 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:37:43.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:43.547158 4175 hybrid_clock.cc:648] HybridClock initialized: now 1790235463547086 us; error 81 us; skew 500 ppm
07:37:43.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:43.548033 4175 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:37:43.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:43.555260 4175 webserver.cc:460] Webserver started at http://127.4.16.254:35349/ using document root <none> and password file <none>
07:37:43.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:43.556262 4175 fs_manager.cc:359] Metadata directory not provided
07:37:43.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:43.556497 4175 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:37:43.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:43.557020 4175 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:37:43.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:43.562016 4175 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-0/data/instance:
07:37:43.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e34879311428453eb5ff4faec6ffd428"
07:37:43.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:37:43 on dist-test-slave-tg41"
07:37:43.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:43.563549 4175 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-0/wal/instance:
07:37:43.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e34879311428453eb5ff4faec6ffd428"
07:37:43.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:37:43 on dist-test-slave-tg41"
07:37:43.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:43.573678 4175 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.000s
07:37:43.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:43.579741 4191 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:37:43.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:43.580880 4175 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.000s sys 0.006s
07:37:43.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:43.581279 4175 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-0/data,/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-0/wal
07:37:43.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e34879311428453eb5ff4faec6ffd428"
07:37:43.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:37:43 on dist-test-slave-tg41"
07:37:43.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:43.581768 4175 fs_report.cc:383] FS layout report
07:37:43.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:37:43.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-0/wal
07:37:43.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-0/wal
07:37:43.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-0/data/data
07:37:43.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:37:43.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:37:43.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:37:43.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:37:43.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:37:43.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:37:43.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:37:43.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:37:43.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:37:43.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:37:43.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:37:43.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:43.630456 4175 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:37:43.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:43.632117 4175 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:37:43.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:43.632638 4175 kserver.cc:163] Server-wide thread pool size limit: 3276
07:37:43.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:43.706631 4175 rpc_server.cc:307] RPC server started. Bound to: 127.4.16.254:41099
07:37:43.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:43.706718 4242 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.16.254:41099 every 8 connection(s)
07:37:43.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:43.709467 4175 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-0/data/info.pb
07:37:43.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:43.710314 4163 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskDxb_lc/build/tsan/bin/kudu as pid 4175
07:37:43.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:43.710665 4163 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-0/wal/instance
07:37:43.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:43.716467 4243 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:37:43.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:43.716430 4163 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskDxb_lc/build/tsan/bin/kudu
07:37:43.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDxb_lc/build/tsan/bin/kudu
07:37:43.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-1/wal
07:37:43.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-1/data
07:37:43.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:37:43.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:37:43.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:37:43.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:37:43.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:37:43.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:37:43.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-1/logs
07:37:43.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-1/data/info.pb
07:37:43.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:37:43.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:37:43.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:37:43.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:37:43.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:37:43.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:37:43.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:37:43.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:37:43.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:37:43.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:37:43.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:37:43.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:37:43.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.16.253:45839
07:37:43.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.16.253
07:37:43.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:37:43.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.16.212:39261
07:37:43.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:37:43.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:37:43.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:37:43.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:37:43.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.16.254:41099,127.4.16.253:45839,127.4.16.252:43219
07:37:43.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:37:43.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34027/jwks.json with env {}
07:37:43.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:43.730427 4243 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.16.254" port: 41099 } has no permanent_uuid. Determining permanent_uuid...
07:37:43.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:43.761942 4243 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.16.253" port: 45839 } has no permanent_uuid. Determining permanent_uuid...
07:37:43.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:43.764550 4192 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.16.253:45839: connect: Connection refused (error 111)
07:37:43.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:43.768522 4243 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.16.253:45839: Network error: Client connection negotiation failed: client connection to 127.4.16.253:45839: connect: Connection refused (error 111)
07:37:43.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:43.818086 4243 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.16.253" port: 45839 } attempt: 1
07:37:43.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:43.822355 4243 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.16.253:45839: Network error: Client connection negotiation failed: client connection to 127.4.16.253:45839: connect: Connection refused (error 111)
07:37:43.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:43.890849 4243 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.16.253" port: 45839 } attempt: 2
07:37:43.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:43.895071 4243 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.16.253:45839: Network error: Client connection negotiation failed: client connection to 127.4.16.253:45839: connect: Connection refused (error 111)
07:37:43.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:43.986532 4243 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.16.253" port: 45839 } attempt: 3
07:37:43.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:43.990713 4243 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.16.253:45839: Network error: Client connection negotiation failed: client connection to 127.4.16.253:45839: connect: Connection refused (error 111)
07:37:44.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:44.052747 4245 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:37:44.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:44.053455 4245 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:37:44.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:44.054138 4245 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:37:44.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:44.087467 4245 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:37:44.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:44.087778 4245 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:37:44.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:44.088009 4245 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:37:44.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:44.088223 4245 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:37:44.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:44.088402 4245 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:37:44.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:44.088528 4245 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34027/jwks.json
07:37:44.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:44.123309 4245 master_runner.cc:386] Master server non-default flags:
07:37:44.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:37:44.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.16.212:39261
07:37:44.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:37:44.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:37:44.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-1/data
07:37:44.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-1/wal
07:37:44.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:37:44.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.16.254:41099,127.4.16.253:45839,127.4.16.252:43219
07:37:44.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:37:44.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:37:44.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:37:44.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.16.253:45839
07:37:44.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:37:44.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:37:44.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:37:44.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34027/jwks.json
07:37:44.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:37:44.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:37:44.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-1/data/info.pb
07:37:44.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.16.253
07:37:44.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:37:44.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:37:44.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:37:44.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:37:44.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:37:44.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:37:44.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-1/logs
07:37:44.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:37:44.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:37:44.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:37:44.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:37:44.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:37:44.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:37:44.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:37:44.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 06:42:32 UTC on ccycloud.quasar-lgarko.root.comops.site
07:37:44.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669406
07:37:44.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:37:44.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:44.124512 4245 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:37:44.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:44.126286 4245 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:37:44.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:44.134140 4243 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.16.253" port: 45839 } attempt: 4
07:37:44.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:44.139616 4243 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.16.253:45839: Network error: Client connection negotiation failed: client connection to 127.4.16.253:45839: connect: Connection refused (error 111)
07:37:44.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:44.145848 4255 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:37:44.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:44.439057 4243 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.16.253" port: 45839 } attempt: 5
07:37:44.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:44.444882 4243 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.16.253:45839: Network error: Client connection negotiation failed: client connection to 127.4.16.253:45839: connect: Connection refused (error 111)
07:37:44.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:44.992481 4243 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.16.253" port: 45839 } attempt: 6
07:37:45.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:45.005505 4243 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.16.253:45839: Network error: Client connection negotiation failed: client connection to 127.4.16.253:45839: connect: Connection refused (error 111)
07:37:45.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:44.145852 4257 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:37:45.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:44.146001 4254 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:37:45.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:45.371387 4256 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1226 milliseconds
07:37:45.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:45.371486 4245 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
07:37:45.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:45.375204 4245 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:37:45.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:45.377861 4245 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:37:45.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:45.379207 4245 hybrid_clock.cc:648] HybridClock initialized: now 1790235465379150 us; error 89 us; skew 500 ppm
07:37:45.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:45.380002 4245 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:37:45.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:45.390594 4245 webserver.cc:460] Webserver started at http://127.4.16.253:40401/ using document root <none> and password file <none>
07:37:45.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:45.391536 4245 fs_manager.cc:359] Metadata directory not provided
07:37:45.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:45.391728 4245 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:37:45.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:45.392149 4245 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:37:45.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:45.396893 4245 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-1/data/instance:
07:37:45.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1a42aaf70bfc4a9c810e4207a7cea8c3"
07:37:45.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:37:45 on dist-test-slave-tg41"
07:37:45.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:45.398082 4245 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-1/wal/instance:
07:37:45.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1a42aaf70bfc4a9c810e4207a7cea8c3"
07:37:45.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:37:45 on dist-test-slave-tg41"
07:37:45.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:45.405545 4245 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.002s
07:37:45.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:45.411413 4266 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:37:45.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:45.412544 4245 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
07:37:45.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:45.412839 4245 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-1/data,/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-1/wal
07:37:45.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1a42aaf70bfc4a9c810e4207a7cea8c3"
07:37:45.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:37:45 on dist-test-slave-tg41"
07:37:45.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:45.413132 4245 fs_report.cc:383] FS layout report
07:37:45.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:37:45.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-1/wal
07:37:45.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-1/wal
07:37:45.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-1/data/data
07:37:45.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:37:45.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:37:45.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:37:45.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:37:45.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:37:45.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:37:45.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:37:45.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:37:45.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:37:45.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:37:45.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:37:45.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:45.465503 4245 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:37:45.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:45.467077 4245 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:37:45.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:45.467494 4245 kserver.cc:163] Server-wide thread pool size limit: 3276
07:37:45.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:45.544232 4245 rpc_server.cc:307] RPC server started. Bound to: 127.4.16.253:45839
07:37:45.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:45.544324 4317 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.16.253:45839 every 8 connection(s)
07:37:45.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:45.548084 4245 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-1/data/info.pb
07:37:45.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:45.552104 4163 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskDxb_lc/build/tsan/bin/kudu as pid 4245
07:37:45.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:45.552594 4163 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-1/wal/instance
07:37:45.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:45.553663 4318 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:37:45.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:45.554912 4163 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskDxb_lc/build/tsan/bin/kudu
07:37:45.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDxb_lc/build/tsan/bin/kudu
07:37:45.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-2/wal
07:37:45.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-2/data
07:37:45.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:37:45.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:37:45.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:37:45.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:37:45.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:37:45.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:37:45.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-2/logs
07:37:45.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-2/data/info.pb
07:37:45.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:37:45.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:37:45.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:37:45.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:37:45.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:37:45.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:37:45.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:37:45.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:37:45.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:37:45.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:37:45.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:37:45.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:37:45.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.16.252:43219
07:37:45.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.16.252
07:37:45.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:37:45.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.16.212:39261
07:37:45.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:37:45.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:37:45.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:37:45.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:37:45.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.16.254:41099,127.4.16.253:45839,127.4.16.252:43219
07:37:45.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:37:45.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34027/jwks.json with env {}
07:37:45.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:45.566152 4318 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.16.254" port: 41099 } has no permanent_uuid. Determining permanent_uuid...
07:37:45.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:45.590310 4318 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.16.253" port: 45839 } has no permanent_uuid. Determining permanent_uuid...
07:37:45.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:45.605831 4318 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.16.252" port: 43219 } has no permanent_uuid. Determining permanent_uuid...
07:37:45.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:45.608639 4270 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.16.252:43219: connect: Connection refused (error 111)
07:37:45.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:45.615903 4318 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.16.252:43219: Network error: Client connection negotiation failed: client connection to 127.4.16.252:43219: connect: Connection refused (error 111)
07:37:45.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:45.665385 4318 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.16.252" port: 43219 } attempt: 1
07:37:45.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:45.670186 4318 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.16.252:43219: Network error: Client connection negotiation failed: client connection to 127.4.16.252:43219: connect: Connection refused (error 111)
07:37:45.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:45.738618 4318 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.16.252" port: 43219 } attempt: 2
07:37:45.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:45.742512 4318 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.16.252:43219: Network error: Client connection negotiation failed: client connection to 127.4.16.252:43219: connect: Connection refused (error 111)
07:37:45.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:45.833956 4318 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.16.252" port: 43219 } attempt: 3
07:37:45.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:45.837572 4318 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.16.252:43219: Network error: Client connection negotiation failed: client connection to 127.4.16.252:43219: connect: Connection refused (error 111)
07:37:45.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:45.882340 4320 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:37:45.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:45.883039 4320 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:37:45.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:45.883773 4320 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:37:45.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:45.915721 4320 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:37:45.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:45.916093 4320 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:37:45.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:45.916419 4320 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:37:45.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:45.916785 4320 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:37:45.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:45.917125 4320 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:37:45.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:45.917385 4320 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34027/jwks.json
07:37:45.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:45.952694 4320 master_runner.cc:386] Master server non-default flags:
07:37:45.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:37:45.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.16.212:39261
07:37:45.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:37:45.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:37:45.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-2/data
07:37:45.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-2/wal
07:37:45.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:37:45.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.16.254:41099,127.4.16.253:45839,127.4.16.252:43219
07:37:45.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:37:45.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:37:45.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:37:45.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.16.252:43219
07:37:45.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:37:45.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:37:45.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:37:45.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34027/jwks.json
07:37:45.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:37:45.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:37:45.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-2/data/info.pb
07:37:45.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.16.252
07:37:45.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:37:45.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:37:45.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:37:45.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:37:45.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:37:45.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:37:45.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-2/logs
07:37:45.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:37:45.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:37:45.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:37:45.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:37:45.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:37:45.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:37:45.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:37:45.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 06:42:32 UTC on ccycloud.quasar-lgarko.root.comops.site
07:37:45.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669406
07:37:45.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:37:45.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:45.954289 4320 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:37:45.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:45.956465 4320 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:37:45.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:45.976234 4331 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:37:45.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:45.981019 4318 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.16.252" port: 43219 } attempt: 4
07:37:45.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:45.985252 4318 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.16.252:43219: Network error: Client connection negotiation failed: client connection to 127.4.16.252:43219: connect: Connection refused (error 111)
07:37:46.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:46.066087 4243 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.16.253" port: 45839 } attempt: 7
07:37:46.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:46.082597 4243 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.16.252" port: 43219 } has no permanent_uuid. Determining permanent_uuid...
07:37:46.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:46.088470 4243 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.16.252:43219: Network error: Client connection negotiation failed: client connection to 127.4.16.252:43219: connect: Connection refused (error 111)
07:37:46.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:46.147099 4243 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.16.252" port: 43219 } attempt: 1
07:37:46.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:46.151221 4243 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.16.252:43219: Network error: Client connection negotiation failed: client connection to 127.4.16.252:43219: connect: Connection refused (error 111)
07:37:46.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:46.232818 4243 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.16.252" port: 43219 } attempt: 2
07:37:46.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:46.237304 4243 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.16.252:43219: Network error: Client connection negotiation failed: client connection to 127.4.16.252:43219: connect: Connection refused (error 111)
07:37:46.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:46.284715 4318 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.16.252" port: 43219 } attempt: 5
07:37:46.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:46.289222 4318 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.16.252:43219: Network error: Client connection negotiation failed: client connection to 127.4.16.252:43219: connect: Connection refused (error 111)
07:37:46.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:46.322904 4243 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.16.252" port: 43219 } attempt: 3
07:37:46.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:46.328161 4243 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.16.252:43219: Network error: Client connection negotiation failed: client connection to 127.4.16.252:43219: connect: Connection refused (error 111)
07:37:46.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:46.468641 4243 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.16.252" port: 43219 } attempt: 4
07:37:46.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:46.473115 4243 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.16.252:43219: Network error: Client connection negotiation failed: client connection to 127.4.16.252:43219: connect: Connection refused (error 111)
07:37:46.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:46.756553 4243 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.16.252" port: 43219 } attempt: 5
07:37:46.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:46.761822 4243 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.16.252:43219: Network error: Client connection negotiation failed: client connection to 127.4.16.252:43219: connect: Connection refused (error 111)
07:37:46.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:46.836624 4318 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.16.252" port: 43219 } attempt: 6
07:37:46.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:46.844434 4318 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.16.252:43219: Network error: Client connection negotiation failed: client connection to 127.4.16.252:43219: connect: Connection refused (error 111)
07:37:47.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:45.977080 4333 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:37:47.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:45.977253 4330 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:37:47.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:45.980078 4320 server_base.cc:1034] running on GCE node
07:37:47.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.192087 4320 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:37:47.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.194887 4320 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:37:47.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.196293 4320 hybrid_clock.cc:648] HybridClock initialized: now 1790235467196264 us; error 42 us; skew 500 ppm
07:37:47.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.197357 4320 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:37:47.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.204239 4320 webserver.cc:460] Webserver started at http://127.4.16.252:42753/ using document root <none> and password file <none>
07:37:47.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.205225 4320 fs_manager.cc:359] Metadata directory not provided
07:37:47.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.205454 4320 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:37:47.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.206046 4320 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:37:47.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.211030 4320 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-2/data/instance:
07:37:47.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6d868c750ef9494d80c2d48599f8c766"
07:37:47.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:37:47 on dist-test-slave-tg41"
07:37:47.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.212235 4320 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-2/wal/instance:
07:37:47.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6d868c750ef9494d80c2d48599f8c766"
07:37:47.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:37:47 on dist-test-slave-tg41"
07:37:47.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.220242 4320 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
07:37:47.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.226408 4344 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:37:47.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.227479 4320 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
07:37:47.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.227833 4320 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-2/data,/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-2/wal
07:37:47.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6d868c750ef9494d80c2d48599f8c766"
07:37:47.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:37:47 on dist-test-slave-tg41"
07:37:47.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.228312 4320 fs_report.cc:383] FS layout report
07:37:47.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:37:47.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-2/wal
07:37:47.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-2/wal
07:37:47.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-2/data/data
07:37:47.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:37:47.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:37:47.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:37:47.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:37:47.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:37:47.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:37:47.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:37:47.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:37:47.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:37:47.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:37:47.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:37:47.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.285027 4320 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:37:47.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.286710 4320 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:37:47.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.287262 4320 kserver.cc:163] Server-wide thread pool size limit: 3276
07:37:47.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.314306 4243 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.16.252" port: 43219 } attempt: 6
07:37:47.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:47.322723 4243 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.16.252:43219: Network error: Client connection negotiation failed: client connection to 127.4.16.252:43219: connect: Connection refused (error 111)
07:37:47.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.368785 4320 rpc_server.cc:307] RPC server started. Bound to: 127.4.16.252:43219
07:37:47.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.368887 4397 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.16.252:43219 every 8 connection(s)
07:37:47.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.372169 4320 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-2/data/info.pb
07:37:47.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.377472 4398 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:37:47.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.381258 4163 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskDxb_lc/build/tsan/bin/kudu as pid 4320
07:37:47.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.381735 4163 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/master-2/wal/instance
07:37:47.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.394248 4398 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.16.254" port: 41099 } has no permanent_uuid. Determining permanent_uuid...
07:37:47.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.426565 4398 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.16.253" port: 45839 } has no permanent_uuid. Determining permanent_uuid...
07:37:47.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.438390 4398 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.16.252" port: 43219 } has no permanent_uuid. Determining permanent_uuid...
07:37:47.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.454900 4398 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6d868c750ef9494d80c2d48599f8c766: Bootstrap starting.
07:37:47.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.461229 4398 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 6d868c750ef9494d80c2d48599f8c766: Neither blocks nor log segments found. Creating new log.
07:37:47.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.463161 4398 log.cc:824] T 00000000000000000000000000000000 P 6d868c750ef9494d80c2d48599f8c766: Log is configured to *not* fsync() on all Append() calls
07:37:47.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.468940 4398 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6d868c750ef9494d80c2d48599f8c766: No bootstrap required, opened a new log
07:37:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.490325 4398 raft_consensus.cc:348] T 00000000000000000000000000000000 P 6d868c750ef9494d80c2d48599f8c766 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e34879311428453eb5ff4faec6ffd428" member_type: VOTER last_known_addr { host: "127.4.16.254" port: 41099 } } peers { permanent_uuid: "1a42aaf70bfc4a9c810e4207a7cea8c3" member_type: VOTER last_known_addr { host: "127.4.16.253" port: 45839 } } peers { permanent_uuid: "6d868c750ef9494d80c2d48599f8c766" member_type: VOTER last_known_addr { host: "127.4.16.252" port: 43219 } }
07:37:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.490963 4398 raft_consensus.cc:374] T 00000000000000000000000000000000 P 6d868c750ef9494d80c2d48599f8c766 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:37:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.491230 4398 raft_consensus.cc:729] T 00000000000000000000000000000000 P 6d868c750ef9494d80c2d48599f8c766 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d868c750ef9494d80c2d48599f8c766, State: Initialized, Role: FOLLOWER
07:37:47.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.492029 4398 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6d868c750ef9494d80c2d48599f8c766 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e34879311428453eb5ff4faec6ffd428" member_type: VOTER last_known_addr { host: "127.4.16.254" port: 41099 } } peers { permanent_uuid: "1a42aaf70bfc4a9c810e4207a7cea8c3" member_type: VOTER last_known_addr { host: "127.4.16.253" port: 45839 } } peers { permanent_uuid: "6d868c750ef9494d80c2d48599f8c766" member_type: VOTER last_known_addr { host: "127.4.16.252" port: 43219 } }
07:37:47.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.494671 4407 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6d868c750ef9494d80c2d48599f8c766 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e34879311428453eb5ff4faec6ffd428" member_type: VOTER last_known_addr { host: "127.4.16.254" port: 41099 } } peers { permanent_uuid: "1a42aaf70bfc4a9c810e4207a7cea8c3" member_type: VOTER last_known_addr { host: "127.4.16.253" port: 45839 } } peers { permanent_uuid: "6d868c750ef9494d80c2d48599f8c766" member_type: VOTER last_known_addr { host: "127.4.16.252" port: 43219 } } }
07:37:47.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.495230 4407 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6d868c750ef9494d80c2d48599f8c766 [sys.catalog]: This master's current role is: FOLLOWER
07:37:47.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.496049 4398 sys_catalog.cc:564] T 00000000000000000000000000000000 P 6d868c750ef9494d80c2d48599f8c766 [sys.catalog]: configured and running, proceeding with master startup.
07:37:47.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:47.523209 4418 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 6d868c750ef9494d80c2d48599f8c766: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:37:47.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:47.523488 4418 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:37:47.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:47.542819 4350 tablet.cc:2369] T 00000000000000000000000000000000 P 6d868c750ef9494d80c2d48599f8c766: Can't schedule compaction. Clean time has not been advanced past its initial value.
07:37:47.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.518584 4320 master_runner.cc:186] Error getting master registration for 127.4.16.254:41099: OK, instance_id { permanent_uuid: "e34879311428453eb5ff4faec6ffd428" instance_seqno: 1790235463685215 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
07:37:47.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.554150 4320 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:37:47.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.594404 4407 raft_consensus.cc:482] T 00000000000000000000000000000000 P 6d868c750ef9494d80c2d48599f8c766 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:37:47.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.594813 4407 raft_consensus.cc:504] T 00000000000000000000000000000000 P 6d868c750ef9494d80c2d48599f8c766 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e34879311428453eb5ff4faec6ffd428" member_type: VOTER last_known_addr { host: "127.4.16.254" port: 41099 } } peers { permanent_uuid: "1a42aaf70bfc4a9c810e4207a7cea8c3" member_type: VOTER last_known_addr { host: "127.4.16.253" port: 45839 } } peers { permanent_uuid: "6d868c750ef9494d80c2d48599f8c766" member_type: VOTER last_known_addr { host: "127.4.16.252" port: 43219 } }
07:37:47.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.597301 4407 leader_election.cc:288] T 00000000000000000000000000000000 P 6d868c750ef9494d80c2d48599f8c766 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e34879311428453eb5ff4faec6ffd428 (127.4.16.254:41099), 1a42aaf70bfc4a9c810e4207a7cea8c3 (127.4.16.253:45839)
07:37:47.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.904767 4318 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.16.252" port: 43219 } attempt: 7
07:37:47.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.926390 4318 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1a42aaf70bfc4a9c810e4207a7cea8c3: Bootstrap starting.
07:37:47.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.932863 4318 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 1a42aaf70bfc4a9c810e4207a7cea8c3: Neither blocks nor log segments found. Creating new log.
07:37:47.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.935082 4318 log.cc:824] T 00000000000000000000000000000000 P 1a42aaf70bfc4a9c810e4207a7cea8c3: Log is configured to *not* fsync() on all Append() calls
07:37:47.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.941175 4318 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1a42aaf70bfc4a9c810e4207a7cea8c3: No bootstrap required, opened a new log
07:37:47.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.974633 4318 raft_consensus.cc:348] T 00000000000000000000000000000000 P 1a42aaf70bfc4a9c810e4207a7cea8c3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e34879311428453eb5ff4faec6ffd428" member_type: VOTER last_known_addr { host: "127.4.16.254" port: 41099 } } peers { permanent_uuid: "1a42aaf70bfc4a9c810e4207a7cea8c3" member_type: VOTER last_known_addr { host: "127.4.16.253" port: 45839 } } peers { permanent_uuid: "6d868c750ef9494d80c2d48599f8c766" member_type: VOTER last_known_addr { host: "127.4.16.252" port: 43219 } }
07:37:47.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.975471 4318 raft_consensus.cc:374] T 00000000000000000000000000000000 P 1a42aaf70bfc4a9c810e4207a7cea8c3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:37:47.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.975740 4318 raft_consensus.cc:729] T 00000000000000000000000000000000 P 1a42aaf70bfc4a9c810e4207a7cea8c3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1a42aaf70bfc4a9c810e4207a7cea8c3, State: Initialized, Role: FOLLOWER
07:37:47.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.976611 4318 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1a42aaf70bfc4a9c810e4207a7cea8c3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e34879311428453eb5ff4faec6ffd428" member_type: VOTER last_known_addr { host: "127.4.16.254" port: 41099 } } peers { permanent_uuid: "1a42aaf70bfc4a9c810e4207a7cea8c3" member_type: VOTER last_known_addr { host: "127.4.16.253" port: 45839 } } peers { permanent_uuid: "6d868c750ef9494d80c2d48599f8c766" member_type: VOTER last_known_addr { host: "127.4.16.252" port: 43219 } }
07:37:47.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.979290 4422 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1a42aaf70bfc4a9c810e4207a7cea8c3 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e34879311428453eb5ff4faec6ffd428" member_type: VOTER last_known_addr { host: "127.4.16.254" port: 41099 } } peers { permanent_uuid: "1a42aaf70bfc4a9c810e4207a7cea8c3" member_type: VOTER last_known_addr { host: "127.4.16.253" port: 45839 } } peers { permanent_uuid: "6d868c750ef9494d80c2d48599f8c766" member_type: VOTER last_known_addr { host: "127.4.16.252" port: 43219 } } }
07:37:47.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.980070 4422 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1a42aaf70bfc4a9c810e4207a7cea8c3 [sys.catalog]: This master's current role is: FOLLOWER
07:37:47.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.981444 4318 sys_catalog.cc:564] T 00000000000000000000000000000000 P 1a42aaf70bfc4a9c810e4207a7cea8c3 [sys.catalog]: configured and running, proceeding with master startup.
07:37:47.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.981645 4293 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6d868c750ef9494d80c2d48599f8c766" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1a42aaf70bfc4a9c810e4207a7cea8c3" is_pre_election: true
07:37:47.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.982504 4293 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 1a42aaf70bfc4a9c810e4207a7cea8c3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d868c750ef9494d80c2d48599f8c766 in term 0.
07:37:47.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.983946 4345 leader_election.cc:302] T 00000000000000000000000000000000 P 6d868c750ef9494d80c2d48599f8c766 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1a42aaf70bfc4a9c810e4207a7cea8c3, 6d868c750ef9494d80c2d48599f8c766; no voters:
07:37:47.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.985030 4407 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 6d868c750ef9494d80c2d48599f8c766 [term 0 FOLLOWER]: Leader pre-election won for term 1
07:37:47.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.985359 4407 raft_consensus.cc:482] T 00000000000000000000000000000000 P 6d868c750ef9494d80c2d48599f8c766 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
07:37:47.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.985677 4407 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 6d868c750ef9494d80c2d48599f8c766 [term 0 FOLLOWER]: Advancing to term 1
07:37:47.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:47.995725 4407 raft_consensus.cc:504] T 00000000000000000000000000000000 P 6d868c750ef9494d80c2d48599f8c766 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e34879311428453eb5ff4faec6ffd428" member_type: VOTER last_known_addr { host: "127.4.16.254" port: 41099 } } peers { permanent_uuid: "1a42aaf70bfc4a9c810e4207a7cea8c3" member_type: VOTER last_known_addr { host: "127.4.16.253" port: 45839 } } peers { permanent_uuid: "6d868c750ef9494d80c2d48599f8c766" member_type: VOTER last_known_addr { host: "127.4.16.252" port: 43219 } }
07:37:48.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.006214 4407 leader_election.cc:288] T 00000000000000000000000000000000 P 6d868c750ef9494d80c2d48599f8c766 [CANDIDATE]: Term 1 election: Requested vote from peers e34879311428453eb5ff4faec6ffd428 (127.4.16.254:41099), 1a42aaf70bfc4a9c810e4207a7cea8c3 (127.4.16.253:45839)
07:37:48.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.007155 4293 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6d868c750ef9494d80c2d48599f8c766" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1a42aaf70bfc4a9c810e4207a7cea8c3"
07:37:48.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.007731 4293 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 1a42aaf70bfc4a9c810e4207a7cea8c3 [term 0 FOLLOWER]: Advancing to term 1
07:37:48.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.014278 4293 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 1a42aaf70bfc4a9c810e4207a7cea8c3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d868c750ef9494d80c2d48599f8c766 in term 1.
07:37:48.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.015636 4345 leader_election.cc:302] T 00000000000000000000000000000000 P 6d868c750ef9494d80c2d48599f8c766 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1a42aaf70bfc4a9c810e4207a7cea8c3, 6d868c750ef9494d80c2d48599f8c766; no voters:
07:37:48.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.016781 4407 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 6d868c750ef9494d80c2d48599f8c766 [term 1 FOLLOWER]: Leader election won for term 1
07:37:48.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.026729 4407 raft_consensus.cc:686] T 00000000000000000000000000000000 P 6d868c750ef9494d80c2d48599f8c766 [term 1 LEADER]: Becoming Leader. State: Replica: 6d868c750ef9494d80c2d48599f8c766, State: Running, Role: LEADER
07:37:48.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.031415 4407 consensus_queue.cc:237] T 00000000000000000000000000000000 P 6d868c750ef9494d80c2d48599f8c766 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e34879311428453eb5ff4faec6ffd428" member_type: VOTER last_known_addr { host: "127.4.16.254" port: 41099 } } peers { permanent_uuid: "1a42aaf70bfc4a9c810e4207a7cea8c3" member_type: VOTER last_known_addr { host: "127.4.16.253" port: 45839 } } peers { permanent_uuid: "6d868c750ef9494d80c2d48599f8c766" member_type: VOTER last_known_addr { host: "127.4.16.252" port: 43219 } }
07:37:48.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.058351 4427 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6d868c750ef9494d80c2d48599f8c766 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6d868c750ef9494d80c2d48599f8c766. Latest consensus state: current_term: 1 leader_uuid: "6d868c750ef9494d80c2d48599f8c766" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e34879311428453eb5ff4faec6ffd428" member_type: VOTER last_known_addr { host: "127.4.16.254" port: 41099 } } peers { permanent_uuid: "1a42aaf70bfc4a9c810e4207a7cea8c3" member_type: VOTER last_known_addr { host: "127.4.16.253" port: 45839 } } peers { permanent_uuid: "6d868c750ef9494d80c2d48599f8c766" member_type: VOTER last_known_addr { host: "127.4.16.252" port: 43219 } } }
07:37:48.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.059126 4427 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6d868c750ef9494d80c2d48599f8c766 [sys.catalog]: This master's current role is: LEADER
07:37:48.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.061214 4433 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
07:37:48.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.075683 4433 catalog_manager.cc:1484] Initializing Kudu cluster ID...
07:37:48.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.102449 4293 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 1a42aaf70bfc4a9c810e4207a7cea8c3 [term 1 FOLLOWER]: Refusing update from remote peer 6d868c750ef9494d80c2d48599f8c766: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
07:37:48.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.105842 4427 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 6d868c750ef9494d80c2d48599f8c766 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a42aaf70bfc4a9c810e4207a7cea8c3" member_type: VOTER last_known_addr { host: "127.4.16.253" port: 45839 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
07:37:48.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:48.121133 4436 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1a42aaf70bfc4a9c810e4207a7cea8c3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:37:48.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:48.121477 4436 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:37:48.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.090348 4245 master_runner.cc:186] Error getting master registration for 127.4.16.254:41099: OK, instance_id { permanent_uuid: "e34879311428453eb5ff4faec6ffd428" instance_seqno: 1790235463685215 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
07:37:48.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.130388 4245 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:37:48.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.151376 4422 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1a42aaf70bfc4a9c810e4207a7cea8c3 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6d868c750ef9494d80c2d48599f8c766. Latest consensus state: current_term: 1 leader_uuid: "6d868c750ef9494d80c2d48599f8c766" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e34879311428453eb5ff4faec6ffd428" member_type: VOTER last_known_addr { host: "127.4.16.254" port: 41099 } } peers { permanent_uuid: "1a42aaf70bfc4a9c810e4207a7cea8c3" member_type: VOTER last_known_addr { host: "127.4.16.253" port: 45839 } } peers { permanent_uuid: "6d868c750ef9494d80c2d48599f8c766" member_type: VOTER last_known_addr { host: "127.4.16.252" port: 43219 } } }
07:37:48.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.152163 4422 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1a42aaf70bfc4a9c810e4207a7cea8c3 [sys.catalog]: This master's current role is: FOLLOWER
07:37:48.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.165115 4439 mvcc.cc:205] Tried to move back new op lower bound from 7332804477315960832 to 7332804477101596672. Current Snapshot: MvccSnapshot[applied={T|T < 7332804477315960832}]
07:37:48.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.173530 4433 catalog_manager.cc:1347] Generated new cluster ID: 65aefdd60390499d92c10b1250f8c883
07:37:48.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.173844 4433 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
07:37:48.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.174158 4407 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6d868c750ef9494d80c2d48599f8c766 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "6d868c750ef9494d80c2d48599f8c766" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e34879311428453eb5ff4faec6ffd428" member_type: VOTER last_known_addr { host: "127.4.16.254" port: 41099 } } peers { permanent_uuid: "1a42aaf70bfc4a9c810e4207a7cea8c3" member_type: VOTER last_known_addr { host: "127.4.16.253" port: 45839 } } peers { permanent_uuid: "6d868c750ef9494d80c2d48599f8c766" member_type: VOTER last_known_addr { host: "127.4.16.252" port: 43219 } } }
07:37:48.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.175305 4407 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6d868c750ef9494d80c2d48599f8c766 [sys.catalog]: This master's current role is: LEADER
07:37:48.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.178627 4422 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1a42aaf70bfc4a9c810e4207a7cea8c3 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "6d868c750ef9494d80c2d48599f8c766" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e34879311428453eb5ff4faec6ffd428" member_type: VOTER last_known_addr { host: "127.4.16.254" port: 41099 } } peers { permanent_uuid: "1a42aaf70bfc4a9c810e4207a7cea8c3" member_type: VOTER last_known_addr { host: "127.4.16.253" port: 45839 } } peers { permanent_uuid: "6d868c750ef9494d80c2d48599f8c766" member_type: VOTER last_known_addr { host: "127.4.16.252" port: 43219 } } }
07:37:48.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.179970 4422 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1a42aaf70bfc4a9c810e4207a7cea8c3 [sys.catalog]: This master's current role is: FOLLOWER
07:37:48.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.247773 4433 catalog_manager.cc:1370] Generated new certificate authority record
07:37:48.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.251219 4433 catalog_manager.cc:1504] Loading token signing keys...
07:37:48.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.289583 4433 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 6d868c750ef9494d80c2d48599f8c766: Generated new TSK 0
07:37:48.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.291574 4433 catalog_manager.cc:1514] Initializing in-progress tserver states...
07:37:48.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.336076 4163 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskDxb_lc/build/tsan/bin/kudu
07:37:48.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDxb_lc/build/tsan/bin/kudu
07:37:48.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-0/wal
07:37:48.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-0/data
07:37:48.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:37:48.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:37:48.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:37:48.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:37:48.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:37:48.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:37:48.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-0/logs
07:37:48.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-0/data/info.pb
07:37:48.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:37:48.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:37:48.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:37:48.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:37:48.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:37:48.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:37:48.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:37:48.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:37:48.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
07:37:48.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:37:48.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.16.193:0
07:37:48.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.16.193
07:37:48.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.16.193
07:37:48.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:37:48.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.16.254:41099,127.4.16.253:45839,127.4.16.252:43219
07:37:48.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.16.212:39261
07:37:48.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:37:48.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:37:48.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
07:37:48.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.356604 4243 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.16.252" port: 43219 } attempt: 7
07:37:48.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.414588 4243 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e34879311428453eb5ff4faec6ffd428: Bootstrap starting.
07:37:48.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.433777 4243 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P e34879311428453eb5ff4faec6ffd428: Neither blocks nor log segments found. Creating new log.
07:37:48.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.440442 4243 log.cc:824] T 00000000000000000000000000000000 P e34879311428453eb5ff4faec6ffd428: Log is configured to *not* fsync() on all Append() calls
07:37:48.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.465132 4243 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e34879311428453eb5ff4faec6ffd428: No bootstrap required, opened a new log
07:37:48.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.580819 4243 raft_consensus.cc:348] T 00000000000000000000000000000000 P e34879311428453eb5ff4faec6ffd428 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e34879311428453eb5ff4faec6ffd428" member_type: VOTER last_known_addr { host: "127.4.16.254" port: 41099 } } peers { permanent_uuid: "1a42aaf70bfc4a9c810e4207a7cea8c3" member_type: VOTER last_known_addr { host: "127.4.16.253" port: 45839 } } peers { permanent_uuid: "6d868c750ef9494d80c2d48599f8c766" member_type: VOTER last_known_addr { host: "127.4.16.252" port: 43219 } }
07:37:48.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.586993 4243 raft_consensus.cc:374] T 00000000000000000000000000000000 P e34879311428453eb5ff4faec6ffd428 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:37:48.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.589341 4243 raft_consensus.cc:729] T 00000000000000000000000000000000 P e34879311428453eb5ff4faec6ffd428 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e34879311428453eb5ff4faec6ffd428, State: Initialized, Role: FOLLOWER
07:37:48.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.594918 4243 consensus_queue.cc:260] T 00000000000000000000000000000000 P e34879311428453eb5ff4faec6ffd428 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e34879311428453eb5ff4faec6ffd428" member_type: VOTER last_known_addr { host: "127.4.16.254" port: 41099 } } peers { permanent_uuid: "1a42aaf70bfc4a9c810e4207a7cea8c3" member_type: VOTER last_known_addr { host: "127.4.16.253" port: 45839 } } peers { permanent_uuid: "6d868c750ef9494d80c2d48599f8c766" member_type: VOTER last_known_addr { host: "127.4.16.252" port: 43219 } }
07:37:48.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.635897 4218 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6d868c750ef9494d80c2d48599f8c766" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e34879311428453eb5ff4faec6ffd428" is_pre_election: true
07:37:48.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.636750 4218 raft_consensus.cc:2454] T 00000000000000000000000000000000 P e34879311428453eb5ff4faec6ffd428 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d868c750ef9494d80c2d48599f8c766 in term 0.
07:37:48.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.637269 4218 rpcz_store.cc:275] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:33290 (request call id 2) took 1039 ms. Trace:
07:37:48.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.637563 4218 rpcz_store.cc:276] 0924 07:37:47.597933 (+ 0us) service_pool.cc:170] Inserting onto call queue
07:37:48.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0924 07:37:47.598252 (+ 319us) service_pool.cc:227] Handling call
07:37:48.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0924 07:37:48.637214 (+1038962us) inbound_call.cc:173] Queueing success response
07:37:48.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
07:37:48.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.638958 4216 raft_consensus.cc:3037] T 00000000000000000000000000000000 P e34879311428453eb5ff4faec6ffd428 [term 0 FOLLOWER]: Advancing to term 1
07:37:48.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.641949 4217 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6d868c750ef9494d80c2d48599f8c766" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e34879311428453eb5ff4faec6ffd428"
07:37:48.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:48.643036 4197 tablet.cc:2369] T 00000000000000000000000000000000 P e34879311428453eb5ff4faec6ffd428: Can't schedule compaction. Clean time has not been advanced past its initial value.
07:37:48.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.651239 4216 raft_consensus.cc:1261] T 00000000000000000000000000000000 P e34879311428453eb5ff4faec6ffd428 [term 1 FOLLOWER]: Refusing update from remote peer 6d868c750ef9494d80c2d48599f8c766: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
07:37:48.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.653141 4407 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 6d868c750ef9494d80c2d48599f8c766 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e34879311428453eb5ff4faec6ffd428" member_type: VOTER last_known_addr { host: "127.4.16.254" port: 41099 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
07:37:48.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.634655 4446 sys_catalog.cc:455] T 00000000000000000000000000000000 P e34879311428453eb5ff4faec6ffd428 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e34879311428453eb5ff4faec6ffd428" member_type: VOTER last_known_addr { host: "127.4.16.254" port: 41099 } } peers { permanent_uuid: "1a42aaf70bfc4a9c810e4207a7cea8c3" member_type: VOTER last_known_addr { host: "127.4.16.253" port: 45839 } } peers { permanent_uuid: "6d868c750ef9494d80c2d48599f8c766" member_type: VOTER last_known_addr { host: "127.4.16.252" port: 43219 } } }
07:37:48.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.654582 4446 sys_catalog.cc:458] T 00000000000000000000000000000000 P e34879311428453eb5ff4faec6ffd428 [sys.catalog]: This master's current role is: FOLLOWER
07:37:48.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.654042 4243 sys_catalog.cc:564] T 00000000000000000000000000000000 P e34879311428453eb5ff4faec6ffd428 [sys.catalog]: configured and running, proceeding with master startup.
07:37:48.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.744279 4449 mvcc.cc:205] Tried to move back new op lower bound from 7332804478021799936 to 7332804477101596672. Current Snapshot: MvccSnapshot[applied={T|T < 7332804477850959872}]
07:37:48.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.752568 4446 sys_catalog.cc:455] T 00000000000000000000000000000000 P e34879311428453eb5ff4faec6ffd428 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6d868c750ef9494d80c2d48599f8c766. Latest consensus state: current_term: 1 leader_uuid: "6d868c750ef9494d80c2d48599f8c766" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e34879311428453eb5ff4faec6ffd428" member_type: VOTER last_known_addr { host: "127.4.16.254" port: 41099 } } peers { permanent_uuid: "1a42aaf70bfc4a9c810e4207a7cea8c3" member_type: VOTER last_known_addr { host: "127.4.16.253" port: 45839 } } peers { permanent_uuid: "6d868c750ef9494d80c2d48599f8c766" member_type: VOTER last_known_addr { host: "127.4.16.252" port: 43219 } } }
07:37:48.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.757885 4446 sys_catalog.cc:458] T 00000000000000000000000000000000 P e34879311428453eb5ff4faec6ffd428 [sys.catalog]: This master's current role is: FOLLOWER
07:37:48.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.754496 4453 sys_catalog.cc:455] T 00000000000000000000000000000000 P e34879311428453eb5ff4faec6ffd428 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "6d868c750ef9494d80c2d48599f8c766" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e34879311428453eb5ff4faec6ffd428" member_type: VOTER last_known_addr { host: "127.4.16.254" port: 41099 } } peers { permanent_uuid: "1a42aaf70bfc4a9c810e4207a7cea8c3" member_type: VOTER last_known_addr { host: "127.4.16.253" port: 45839 } } peers { permanent_uuid: "6d868c750ef9494d80c2d48599f8c766" member_type: VOTER last_known_addr { host: "127.4.16.252" port: 43219 } } }
07:37:48.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.759923 4453 sys_catalog.cc:458] T 00000000000000000000000000000000 P e34879311428453eb5ff4faec6ffd428 [sys.catalog]: This master's current role is: FOLLOWER
07:37:48.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.829526 4466 catalog_manager.cc:1259] Loaded cluster ID: 65aefdd60390499d92c10b1250f8c883
07:37:48.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.832693 4466 catalog_manager.cc:1552] T 00000000000000000000000000000000 P e34879311428453eb5ff4faec6ffd428: loading cluster ID for follower catalog manager: success
07:37:48.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.843158 4466 catalog_manager.cc:1574] T 00000000000000000000000000000000 P e34879311428453eb5ff4faec6ffd428: acquiring CA information for follower catalog manager: success
07:37:48.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:48.847246 4443 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:37:48.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:48.848223 4443 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:37:48.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:48.849220 4443 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:37:48.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.849494 4466 catalog_manager.cc:1602] T 00000000000000000000000000000000 P e34879311428453eb5ff4faec6ffd428: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
07:37:48.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:48.906230 4443 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:37:48.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:48.908090 4443 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.4.16.193
07:37:48.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.968878 4443 tablet_server_runner.cc:78] Tablet server non-default flags:
07:37:48.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:37:48.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.16.212:39261
07:37:48.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:37:48.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:37:48.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-0/data
07:37:48.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-0/wal
07:37:48.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:37:48.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:37:48.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.16.193:0
07:37:48.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:37:48.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:37:48.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:37:48.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-0/data/info.pb
07:37:48.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.16.193
07:37:48.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:37:48.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.16.254:41099,127.4.16.253:45839,127.4.16.252:43219
07:37:48.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:37:48.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:37:48.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:37:48.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:37:48.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:37:48.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.16.193
07:37:48.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-0/logs
07:37:48.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:37:48.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:37:48.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:37:48.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
07:37:48.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:37:48.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:37:48.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:37:48.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 06:42:32 UTC on ccycloud.quasar-lgarko.root.comops.site
07:37:48.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669406
07:37:48.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:37:48.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.972137 4443 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:37:48.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:48.974431 4443 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:37:49.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:49.020731 4470 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:37:49.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:49.126947 4436 catalog_manager.cc:1259] Loaded cluster ID: 65aefdd60390499d92c10b1250f8c883
07:37:49.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:49.127627 4436 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 1a42aaf70bfc4a9c810e4207a7cea8c3: loading cluster ID for follower catalog manager: success
07:37:49.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:49.148274 4436 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 1a42aaf70bfc4a9c810e4207a7cea8c3: acquiring CA information for follower catalog manager: success
07:37:49.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:49.154253 4436 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 1a42aaf70bfc4a9c810e4207a7cea8c3: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
07:37:49.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:49.285998 4407 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6d868c750ef9494d80c2d48599f8c766 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "6d868c750ef9494d80c2d48599f8c766" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e34879311428453eb5ff4faec6ffd428" member_type: VOTER last_known_addr { host: "127.4.16.254" port: 41099 } } peers { permanent_uuid: "1a42aaf70bfc4a9c810e4207a7cea8c3" member_type: VOTER last_known_addr { host: "127.4.16.253" port: 45839 } } peers { permanent_uuid: "6d868c750ef9494d80c2d48599f8c766" member_type: VOTER last_known_addr { host: "127.4.16.252" port: 43219 } } }
07:37:49.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:49.287416 4407 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6d868c750ef9494d80c2d48599f8c766 [sys.catalog]: This master's current role is: LEADER
07:37:50.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:49.030378 4469 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:37:50.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:49.031481 4443 server_base.cc:1034] running on GCE node
07:37:50.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:49.024269 4473 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:37:50.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:50.567592 4443 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:37:50.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:50.578289 4443 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:37:50.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:50.582679 4443 hybrid_clock.cc:648] HybridClock initialized: now 1790235470582628 us; error 60 us; skew 500 ppm
07:37:50.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:50.583847 4443 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:37:50.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:50.618559 4443 webserver.cc:460] Webserver started at http://127.4.16.193:35253/ using document root <none> and password file <none>
07:37:50.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:50.620250 4443 fs_manager.cc:359] Metadata directory not provided
07:37:50.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:50.620744 4443 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:37:50.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:50.621501 4443 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:37:50.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:50.628996 4443 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-0/data/instance:
07:37:50.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "215168b9916f4c02ab04b429c19b2a04"
07:37:50.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:37:50 on dist-test-slave-tg41"
07:37:50.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:50.630877 4443 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-0/wal/instance:
07:37:50.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "215168b9916f4c02ab04b429c19b2a04"
07:37:50.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:37:50 on dist-test-slave-tg41"
07:37:50.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:50.646965 4443 fs_manager.cc:691] Time spent creating directory manager: real 0.015s user 0.009s sys 0.004s
07:37:50.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:50.662979 4482 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:37:50.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:50.664649 4443 fs_manager.cc:725] Time spent opening block manager: real 0.010s user 0.008s sys 0.000s
07:37:50.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:50.665253 4443 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-0/data,/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-0/wal
07:37:50.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "215168b9916f4c02ab04b429c19b2a04"
07:37:50.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:37:50 on dist-test-slave-tg41"
07:37:50.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:50.665959 4443 fs_report.cc:383] FS layout report
07:37:50.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:37:50.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-0/wal
07:37:50.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-0/wal
07:37:50.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-0/data/data
07:37:50.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:37:50.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:37:50.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:37:50.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:37:50.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:37:50.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:37:50.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:37:50.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:37:50.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:37:50.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:37:50.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:37:50.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:50.743044 4443 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:37:50.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:50.750473 4443 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:37:50.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:50.751384 4443 kserver.cc:163] Server-wide thread pool size limit: 3276
07:37:50.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:50.774355 4443 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
07:37:50.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:50.799000 4443 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
07:37:50.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:50.801447 4443 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.002s user 0.000s sys 0.000s
07:37:50.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:50.802153 4443 ts_tablet_manager.cc:610] Registered 0 tablets
07:37:50.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:50.802661 4443 ts_tablet_manager.cc:589] Time spent register tablets: real 0.001s user 0.000s sys 0.000s
07:37:51.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:51.239277 4443 rpc_server.cc:307] RPC server started. Bound to: 127.4.16.193:36801
07:37:51.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:51.239444 4594 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.16.193:36801 every 8 connection(s)
07:37:51.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:51.242130 4443 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-0/data/info.pb
07:37:51.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:51.250766 4163 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskDxb_lc/build/tsan/bin/kudu as pid 4443
07:37:51.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:51.251205 4163 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-0/wal/instance
07:37:51.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:51.283923 4163 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskDxb_lc/build/tsan/bin/kudu
07:37:51.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDxb_lc/build/tsan/bin/kudu
07:37:51.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-1/wal
07:37:51.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-1/data
07:37:51.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:37:51.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:37:51.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:37:51.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:37:51.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:37:51.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:37:51.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-1/logs
07:37:51.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-1/data/info.pb
07:37:51.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:37:51.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:37:51.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:37:51.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:37:51.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:37:51.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:37:51.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:37:51.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:37:51.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
07:37:51.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:37:51.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.16.194:0
07:37:51.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.16.194
07:37:51.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.16.194
07:37:51.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:37:51.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.16.254:41099,127.4.16.253:45839,127.4.16.252:43219
07:37:51.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.16.212:39261
07:37:51.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:37:51.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:37:51.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
07:37:51.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:51.324831 4596 heartbeater.cc:347] Connected to a master server at 127.4.16.253:45839
07:37:51.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:51.325323 4596 heartbeater.cc:464] Registering TS with master...
07:37:51.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:51.326763 4596 heartbeater.cc:511] Master 127.4.16.253:45839 requested a full tablet report, sending...
07:37:51.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:51.330561 4283 ts_manager.cc:194] Registered new tserver with Master: 215168b9916f4c02ab04b429c19b2a04 (127.4.16.193:36801)
07:37:51.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:51.330950 4595 heartbeater.cc:347] Connected to a master server at 127.4.16.252:43219
07:37:51.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:51.331272 4595 heartbeater.cc:464] Registering TS with master...
07:37:51.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:51.332020 4595 heartbeater.cc:511] Master 127.4.16.252:43219 requested a full tablet report, sending...
07:37:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:51.335788 4362 ts_manager.cc:194] Registered new tserver with Master: 215168b9916f4c02ab04b429c19b2a04 (127.4.16.193:36801)
07:37:51.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:51.339170 4362 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.4.16.193:45365
07:37:51.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:51.339735 4597 heartbeater.cc:347] Connected to a master server at 127.4.16.254:41099
07:37:51.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:51.340060 4597 heartbeater.cc:464] Registering TS with master...
07:37:51.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:51.340888 4597 heartbeater.cc:511] Master 127.4.16.254:41099 requested a full tablet report, sending...
07:37:51.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:51.344134 4208 ts_manager.cc:194] Registered new tserver with Master: 215168b9916f4c02ab04b429c19b2a04 (127.4.16.193:36801)
07:37:51.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:51.651316 4603 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:37:51.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:51.651887 4603 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:37:51.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:51.652500 4603 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:37:51.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:51.684067 4603 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:37:51.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:51.685036 4603 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.4.16.194
07:37:51.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:51.719547 4603 tablet_server_runner.cc:78] Tablet server non-default flags:
07:37:51.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:37:51.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.16.212:39261
07:37:51.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:37:51.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:37:51.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-1/data
07:37:51.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-1/wal
07:37:51.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:37:51.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:37:51.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.16.194:0
07:37:51.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:37:51.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:37:51.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:37:51.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-1/data/info.pb
07:37:51.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.16.194
07:37:51.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:37:51.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.16.254:41099,127.4.16.253:45839,127.4.16.252:43219
07:37:51.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:37:51.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:37:51.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:37:51.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:37:51.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:37:51.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.16.194
07:37:51.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-1/logs
07:37:51.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:37:51.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:37:51.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:37:51.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
07:37:51.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:37:51.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:37:51.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:37:51.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 06:42:32 UTC on ccycloud.quasar-lgarko.root.comops.site
07:37:51.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669406
07:37:51.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:37:51.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:51.721115 4603 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:37:51.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:51.723376 4603 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:37:51.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:51.744714 4612 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:37:52.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:52.344228 4595 heartbeater.cc:503] Master 127.4.16.252:43219 was elected leader, sending a full tablet report...
07:37:52.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:51.744700 4611 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:37:52.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:51.747069 4603 server_base.cc:1034] running on GCE node
07:37:52.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:51.744719 4614 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:37:52.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:52.942577 4603 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:37:52.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:52.944859 4603 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:37:52.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:52.946283 4603 hybrid_clock.cc:648] HybridClock initialized: now 1790235472946246 us; error 52 us; skew 500 ppm
07:37:52.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:52.947304 4603 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:37:52.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:52.954257 4603 webserver.cc:460] Webserver started at http://127.4.16.194:36075/ using document root <none> and password file <none>
07:37:52.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:52.955296 4603 fs_manager.cc:359] Metadata directory not provided
07:37:52.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:52.955543 4603 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:37:52.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:52.956073 4603 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:37:52.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:52.961244 4603 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-1/data/instance:
07:37:52.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1088a58c483a4c1eb305e05235de270a"
07:37:52.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:37:52 on dist-test-slave-tg41"
07:37:52.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:52.962633 4603 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-1/wal/instance:
07:37:52.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1088a58c483a4c1eb305e05235de270a"
07:37:52.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:37:52 on dist-test-slave-tg41"
07:37:52.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:52.970197 4603 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.002s sys 0.005s
07:37:52.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:52.975669 4624 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:37:52.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:52.976714 4603 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.005s sys 0.000s
07:37:52.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:52.977118 4603 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-1/data,/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-1/wal
07:37:52.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1088a58c483a4c1eb305e05235de270a"
07:37:52.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:37:52 on dist-test-slave-tg41"
07:37:52.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:52.977475 4603 fs_report.cc:383] FS layout report
07:37:52.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:37:52.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-1/wal
07:37:52.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-1/wal
07:37:52.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-1/data/data
07:37:52.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:37:52.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:37:52.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:37:52.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:37:52.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:37:52.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:37:52.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:37:52.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:37:52.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:37:52.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:37:52.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:37:53.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:53.015501 4603 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:37:53.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:53.017057 4603 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:37:53.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:53.017575 4603 kserver.cc:163] Server-wide thread pool size limit: 3276
07:37:53.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:53.020763 4603 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
07:37:53.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:53.024963 4603 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
07:37:53.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:53.025266 4603 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
07:37:53.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:53.025594 4603 ts_tablet_manager.cc:610] Registered 0 tablets
07:37:53.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:53.025825 4603 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.001s
07:37:53.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:53.176214 4603 rpc_server.cc:307] RPC server started. Bound to: 127.4.16.194:44693
07:37:53.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:53.176352 4736 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.16.194:44693 every 8 connection(s)
07:37:53.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:53.178701 4603 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-1/data/info.pb
07:37:53.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:53.189965 4163 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskDxb_lc/build/tsan/bin/kudu as pid 4603
07:37:53.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:53.190779 4163 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-1/wal/instance
07:37:53.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:53.207861 4163 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskDxb_lc/build/tsan/bin/kudu
07:37:53.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDxb_lc/build/tsan/bin/kudu
07:37:53.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-2/wal
07:37:53.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-2/data
07:37:53.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:37:53.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:37:53.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:37:53.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:37:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:37:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:37:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-2/logs
07:37:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-2/data/info.pb
07:37:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:37:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:37:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:37:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:37:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:37:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:37:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:37:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:37:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
07:37:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:37:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.16.195:0
07:37:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.16.195
07:37:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.16.195
07:37:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:37:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.16.254:41099,127.4.16.253:45839,127.4.16.252:43219
07:37:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.16.212:39261
07:37:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:37:53.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:37:53.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
07:37:53.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:53.219801 4737 heartbeater.cc:347] Connected to a master server at 127.4.16.252:43219
07:37:53.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:53.220276 4737 heartbeater.cc:464] Registering TS with master...
07:37:53.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:53.221252 4737 heartbeater.cc:511] Master 127.4.16.252:43219 requested a full tablet report, sending...
07:37:53.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:53.223718 4362 ts_manager.cc:194] Registered new tserver with Master: 1088a58c483a4c1eb305e05235de270a (127.4.16.194:44693)
07:37:53.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:53.225876 4362 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.4.16.194:42411
07:37:53.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:53.255787 4739 heartbeater.cc:347] Connected to a master server at 127.4.16.254:41099
07:37:53.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:53.256119 4739 heartbeater.cc:464] Registering TS with master...
07:37:53.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:53.256793 4739 heartbeater.cc:511] Master 127.4.16.254:41099 requested a full tablet report, sending...
07:37:53.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:53.259049 4738 heartbeater.cc:347] Connected to a master server at 127.4.16.253:45839
07:37:53.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:53.259364 4738 heartbeater.cc:464] Registering TS with master...
07:37:53.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:53.259440 4208 ts_manager.cc:194] Registered new tserver with Master: 1088a58c483a4c1eb305e05235de270a (127.4.16.194:44693)
07:37:53.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:53.260579 4738 heartbeater.cc:511] Master 127.4.16.253:45839 requested a full tablet report, sending...
07:37:53.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:53.262842 4283 ts_manager.cc:194] Registered new tserver with Master: 1088a58c483a4c1eb305e05235de270a (127.4.16.194:44693)
07:37:53.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:53.557991 4746 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:37:53.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:53.558581 4746 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:37:53.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:53.559015 4746 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:37:53.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:53.592572 4746 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:37:53.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:53.593410 4746 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.4.16.195
07:37:53.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:53.634394 4746 tablet_server_runner.cc:78] Tablet server non-default flags:
07:37:53.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:37:53.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.16.212:39261
07:37:53.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:37:53.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:37:53.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-2/data
07:37:53.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-2/wal
07:37:53.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:37:53.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:37:53.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.16.195:0
07:37:53.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:37:53.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:37:53.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:37:53.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-2/data/info.pb
07:37:53.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.16.195
07:37:53.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:37:53.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.16.254:41099,127.4.16.253:45839,127.4.16.252:43219
07:37:53.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:37:53.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:37:53.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:37:53.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:37:53.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:37:53.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.16.195
07:37:53.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-2/logs
07:37:53.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:37:53.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:37:53.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:37:53.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
07:37:53.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:37:53.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:37:53.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:37:53.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 06:42:32 UTC on ccycloud.quasar-lgarko.root.comops.site
07:37:53.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669406
07:37:53.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:37:53.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:53.635607 4746 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:37:53.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:53.637382 4746 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:37:53.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:53.659102 4755 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:37:54.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:54.232229 4737 heartbeater.cc:503] Master 127.4.16.252:43219 was elected leader, sending a full tablet report...
07:37:54.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:53.659168 4753 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:37:54.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:53.659660 4752 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:37:54.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:53.661438 4746 server_base.cc:1034] running on GCE node
07:37:54.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:54.879446 4746 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:37:54.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:54.881596 4746 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:37:54.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:54.883054 4746 hybrid_clock.cc:648] HybridClock initialized: now 1790235474883024 us; error 46 us; skew 500 ppm
07:37:54.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:54.883924 4746 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:37:54.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:54.890254 4746 webserver.cc:460] Webserver started at http://127.4.16.195:37301/ using document root <none> and password file <none>
07:37:54.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:54.891302 4746 fs_manager.cc:359] Metadata directory not provided
07:37:54.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:54.891534 4746 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:37:54.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:54.892081 4746 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:37:54.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:54.897053 4746 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-2/data/instance:
07:37:54.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "47341ec50ff54ecfa29dad4d8c48ac93"
07:37:54.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:37:54 on dist-test-slave-tg41"
07:37:54.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:54.898298 4746 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-2/wal/instance:
07:37:54.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "47341ec50ff54ecfa29dad4d8c48ac93"
07:37:54.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:37:54 on dist-test-slave-tg41"
07:37:54.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:54.905558 4746 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
07:37:54.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:54.911038 4763 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:37:54.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:54.911980 4746 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.005s sys 0.000s
07:37:54.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:54.912365 4746 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-2/data,/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-2/wal
07:37:54.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "47341ec50ff54ecfa29dad4d8c48ac93"
07:37:54.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:37:54 on dist-test-slave-tg41"
07:37:54.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:54.912765 4746 fs_report.cc:383] FS layout report
07:37:54.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:37:54.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-2/wal
07:37:54.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-2/wal
07:37:54.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-2/data/data
07:37:54.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:37:54.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:37:54.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:37:54.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:37:54.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:37:54.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:37:54.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:37:54.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:37:54.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:37:54.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:37:54.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:37:54.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:54.963496 4746 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:37:54.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:54.964968 4746 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:37:54.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:54.965382 4746 kserver.cc:163] Server-wide thread pool size limit: 3276
07:37:54.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:54.967897 4746 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
07:37:54.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:54.971958 4746 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
07:37:54.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:54.972177 4746 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
07:37:54.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:54.972363 4746 ts_tablet_manager.cc:610] Registered 0 tablets
07:37:54.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:54.972493 4746 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
07:37:55.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:55.126425 4746 rpc_server.cc:307] RPC server started. Bound to: 127.4.16.195:40803
07:37:55.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:55.126503 4875 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.16.195:40803 every 8 connection(s)
07:37:55.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:55.128886 4746 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-2/data/info.pb
07:37:55.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:55.131513 4163 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskDxb_lc/build/tsan/bin/kudu as pid 4746
07:37:55.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:55.132094 4163 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster5006311552326789968/ts-2/wal/instance
07:37:55.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:55.189577 4877 heartbeater.cc:347] Connected to a master server at 127.4.16.253:45839
07:37:55.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:55.190083 4877 heartbeater.cc:464] Registering TS with master...
07:37:55.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:55.191283 4877 heartbeater.cc:511] Master 127.4.16.253:45839 requested a full tablet report, sending...
07:37:55.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:55.193434 4876 heartbeater.cc:347] Connected to a master server at 127.4.16.252:43219
07:37:55.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:55.193487 4878 heartbeater.cc:347] Connected to a master server at 127.4.16.254:41099
07:37:55.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:55.193845 4876 heartbeater.cc:464] Registering TS with master...
07:37:55.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:55.194096 4878 heartbeater.cc:464] Registering TS with master...
07:37:55.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:55.194654 4876 heartbeater.cc:511] Master 127.4.16.252:43219 requested a full tablet report, sending...
07:37:55.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:55.194820 4878 heartbeater.cc:511] Master 127.4.16.254:41099 requested a full tablet report, sending...
07:37:55.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:55.196764 4362 ts_manager.cc:194] Registered new tserver with Master: 47341ec50ff54ecfa29dad4d8c48ac93 (127.4.16.195:40803)
07:37:55.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:55.197369 4208 ts_manager.cc:194] Registered new tserver with Master: 47341ec50ff54ecfa29dad4d8c48ac93 (127.4.16.195:40803)
07:37:55.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:55.197443 4283 ts_manager.cc:194] Registered new tserver with Master: 47341ec50ff54ecfa29dad4d8c48ac93 (127.4.16.195:40803)
07:37:55.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:55.198948 4362 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.4.16.195:44939
07:37:55.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:55.219888 4163 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
07:37:55.231 [DEBUG - main] (MiniKuduCluster.java:195) Response:
07:37:55.233 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
07:37:55.244 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.4.16.254"
port: 41099
}
bound_http_address {
host: "127.4.16.254"
port: 35349
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.4.16.253"
port: 45839
}
bound_http_address {
host: "127.4.16.253"
port: 40401
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.4.16.252"
port: 43219
}
bound_http_address {
host: "127.4.16.252"
port: 42753
}
}
}
07:37:55.250 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
07:37:55.253 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.4.16.193"
port: 36801
}
bound_http_address {
host: "127.4.16.193"
port: 35253
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.4.16.194"
port: 44693
}
bound_http_address {
host: "127.4.16.194"
port: 36075
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.4.16.195"
port: 40803
}
bound_http_address {
host: "127.4.16.195"
port: 37301
}
}
}
07:37:55.253 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
07:37:55.432 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
07:37:55.538 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskDxb_lc/test-tmp
07:37:55.538 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
07:37:55.539 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskDxb_lc/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
07:37:55.541 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
07:37:55.916 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:37:55.917 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
07:37:55.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:55.920213 4887 mini_oidc.cc:152] Starting JWKS server
07:37:55.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:55.924413 4887 webserver.cc:460] Webserver started at http://127.0.0.1:44963/ using document root <none> and password file <none>
07:37:55.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:55.925004 4887 mini_oidc.cc:189] Starting OIDC Discovery server
07:37:55.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:55.927212 4887 webserver.cc:460] Webserver started at http://127.0.0.1:42461/ using document root <none> and password file <none>
07:37:56.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:37:56.202137 4876 heartbeater.cc:503] Master 127.4.16.252:43219 was elected leader, sending a full tablet report...
07:37:56.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
07:37:56.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/krb5kdc/principal' for realm 'KRBTEST.COM',
07:37:56.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
07:37:56.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:37:56 dist-test-slave-tg41 krb5kdc[4900](info): setting up network...
07:37:56.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
07:37:56.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:37:56 dist-test-slave-tg41 krb5kdc[4900](info): set up 2 sockets
07:37:56.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:37:56 dist-test-slave-tg41 krb5kdc[4900](info): commencing operation
07:37:56.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
07:37:57.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:57.940497 4887 mini_kdc.cc:121] Time spent starting KDC: real 1.988s user 0.012s sys 0.010s
07:37:57.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
07:37:58.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
07:37:58.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
07:37:58.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:58.192303 4887 mini_kdc.cc:253] Time spent creating user principal test-admin: real 0.251s user 0.003s sys 0.004s
07:37:58.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
07:37:58.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
07:37:58.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
07:37:58.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:58.434509 4887 mini_kdc.cc:253] Time spent creating user principal test-user: real 0.242s user 0.001s sys 0.004s
07:37:58.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
07:37:58.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
07:37:58.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
07:37:58.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:58.733001 4887 mini_kdc.cc:253] Time spent creating user principal joe-interloper: real 0.298s user 0.001s sys 0.004s
07:37:59.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
07:37:59.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/krb5kdc/test-user.keytab.
07:37:59.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/krb5kdc/test-user.keytab.
07:37:59.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:37:59.568990 4887 mini_kdc.cc:302] Time spent creating keytab for test-user: real 0.836s user 0.005s sys 0.000s
07:37:59.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:37:59 dist-test-slave-tg41 krb5kdc[4900](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790235479, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:37:59.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
07:37:59.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-24T07:37:59Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
07:37:59.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-24T07:37:59Z Disabled control of system clock
07:37:59.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.4.197.254@KRBTEST.COM; defaulting to no policy
07:37:59.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:37:59.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.4.197.254@KRBTEST.COM" created.
07:38:00.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:38:00.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.197.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/krb5kdc/kudu_127.4.197.254.keytab.
07:38:00.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.197.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/krb5kdc/kudu_127.4.197.254.keytab.
07:38:00.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:00.128619 4887 mini_kdc.cc:262] Time spent creating service keytab for kudu/127.4.197.254: real 0.511s user 0.000s sys 0.009s
07:38:00.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:00.131103 4887 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskDxb_lc/build/tsan/bin/kudu
07:38:00.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDxb_lc/build/tsan/bin/kudu
07:38:00.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-0/wal
07:38:00.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-0/data
07:38:00.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:38:00.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:38:00.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:38:00.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:38:00.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:38:00.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:38:00.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-0/logs
07:38:00.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-0/data/info.pb
07:38:00.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:38:00.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:38:00.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:38:00.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:38:00.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:38:00.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:38:00.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:38:00.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:38:00.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:38:00.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:38:00.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:38:00.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:38:00.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.197.254:40799
07:38:00.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.197.254
07:38:00.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:38:00.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/krb5kdc/kudu_127.4.197.254.keytab
07:38:00.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.197.254
07:38:00.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:38:00.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:38:00.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:38:00.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.197.212:38399
07:38:00.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:38:00.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:38:00.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:38:00.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:38:00.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.197.254:40799,127.4.197.253:37597,127.4.197.252:37301
07:38:00.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:38:00.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44963/jwks.json
07:38:00.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:38:00.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:00.324246 4394 debug-util.cc:398] Leaking SignalData structure 0x7b08000908a0 after lost signal to thread 4325
07:38:00.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:00.324903 4394 debug-util.cc:398] Leaking SignalData structure 0x7b08000b0ba0 after lost signal to thread 4397
07:38:00.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:00.464618 4915 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:38:00.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:00.465162 4915 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:38:00.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:00.466063 4915 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:38:00.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:00.503626 4915 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:38:00.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:00.503852 4915 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:38:00.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:00.504117 4915 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:38:00.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:00.504307 4915 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:38:00.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:00.504498 4915 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:38:00.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:00.504698 4915 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:38:00.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:00.504837 4915 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44963/jwks.json
07:38:00.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:00.542402 4915 master_runner.cc:386] Master server non-default flags:
07:38:00.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:38:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.197.212:38399
07:38:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:38:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:38:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-0/data
07:38:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-0/wal
07:38:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:38:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:38:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.197.254:40799,127.4.197.253:37597,127.4.197.252:37301
07:38:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/krb5kdc/kudu_127.4.197.254.keytab
07:38:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.197.254
07:38:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:38:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:38:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:38:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.197.254:40799
07:38:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:38:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:38:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:38:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44963/jwks.json
07:38:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:38:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:38:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:38:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:38:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:38:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-0/data/info.pb
07:38:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.197.254
07:38:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:38:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:38:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:38:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:38:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:38:00.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:38:00.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-0/logs
07:38:00.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:38:00.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:38:00.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:38:00.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:38:00.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:38:00.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:38:00.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:38:00.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 06:42:32 UTC on ccycloud.quasar-lgarko.root.comops.site
07:38:00.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669406
07:38:00.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:38:00.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:00.543655 4915 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:38:00.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:00.545029 4915 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:38:00.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:00.569273 4921 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:38:02.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:02.249810 4872 debug-util.cc:398] Leaking SignalData structure 0x7b08000aff60 after lost signal to thread 4747
07:38:02.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:02.250893 4872 debug-util.cc:398] Leaking SignalData structure 0x7b08000b59a0 after lost signal to thread 4875
07:38:02.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:01.962287 4920 debug-util.cc:398] Leaking SignalData structure 0x7b0800014920 after lost signal to thread 4915
07:38:02.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:02.362419 4915 thread.cc:636] GCE (cloud detector) Time spent creating pthread: real 1.796s user 0.000s sys 0.002s
07:38:02.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:02.363719 4915 thread.cc:602] GCE (cloud detector) Time spent starting thread: real 1.797s user 0.000s sys 0.003s
07:38:02.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:00.570407 4922 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:38:02.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:02.371948 4915 server_base.cc:1034] running on GCE node
07:38:02.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:02.373412 4929 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:38:02.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:02.374927 4915 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:38:02.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:02.377985 4915 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:38:02.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:02.379601 4915 hybrid_clock.cc:648] HybridClock initialized: now 1790235482379560 us; error 30 us; skew 500 ppm
07:38:02.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:02.380720 4915 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:38:02.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:38:02 dist-test-slave-tg41 krb5kdc[4900](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790235482, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.197.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:38:02.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:02.393476 4915 init.cc:377] Logged in from keytab as kudu/127.4.197.254@KRBTEST.COM (short username kudu)
07:38:02.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:02.399887 4915 webserver.cc:460] Webserver started at http://127.4.197.254:39931/ using document root <none> and password file <none>
07:38:02.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:02.400993 4915 fs_manager.cc:359] Metadata directory not provided
07:38:02.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:02.401239 4915 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:38:02.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:02.401818 4915 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:38:02.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:02.406818 4915 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-0/data/instance:
07:38:02.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "925836e479bc498db068663e717e7b91"
07:38:02.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:38:02 on dist-test-slave-tg41"
07:38:02.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:02.407887 4915 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-0/wal/instance:
07:38:02.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "925836e479bc498db068663e717e7b91"
07:38:02.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:38:02 on dist-test-slave-tg41"
07:38:02.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:02.416371 4915 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.005s sys 0.004s
07:38:02.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:02.422046 4935 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:38:02.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:02.423202 4915 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
07:38:02.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:02.423563 4915 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-0/data,/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-0/wal
07:38:02.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "925836e479bc498db068663e717e7b91"
07:38:02.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:38:02 on dist-test-slave-tg41"
07:38:02.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:02.423911 4915 fs_report.cc:383] FS layout report
07:38:02.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:38:02.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-0/wal
07:38:02.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-0/wal
07:38:02.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-0/data/data
07:38:02.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:38:02.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:38:02.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:38:02.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:38:02.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:38:02.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:38:02.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:38:02.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:38:02.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:38:02.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:38:02.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:38:02.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:02.471844 4915 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:38:02.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:02.478047 4915 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:38:02.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:02.478583 4915 kserver.cc:163] Server-wide thread pool size limit: 3276
07:38:02.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:02.581768 4915 rpc_server.cc:307] RPC server started. Bound to: 127.4.197.254:40799
07:38:02.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:02.581841 4986 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.197.254:40799 every 8 connection(s)
07:38:02.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:02.585250 4915 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-0/data/info.pb
07:38:02.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:02.586223 4887 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskDxb_lc/build/tsan/bin/kudu as pid 4915
07:38:02.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:02.587239 4887 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-0/wal/instance
07:38:02.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:02.599417 4987 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:38:02.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.4.197.253@KRBTEST.COM; defaulting to no policy
07:38:02.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:38:02.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.4.197.253@KRBTEST.COM" created.
07:38:02.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:02.619071 4987 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.197.254" port: 40799 } has no permanent_uuid. Determining permanent_uuid...
07:38:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:38:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.197.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/krb5kdc/kudu_127.4.197.253.keytab.
07:38:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.197.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/krb5kdc/kudu_127.4.197.253.keytab.
07:38:02.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:02.645746 4887 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskDxb_lc/build/tsan/bin/kudu
07:38:02.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDxb_lc/build/tsan/bin/kudu
07:38:02.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-1/wal
07:38:02.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-1/data
07:38:02.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:38:02.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:38:02.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:38:02.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:38:02.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:38:02.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:38:02.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-1/logs
07:38:02.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-1/data/info.pb
07:38:02.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:38:02.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:38:02.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:38:02.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:38:02.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:38:02.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:38:02.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:38:02.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:38:02.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:38:02.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:38:02.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:38:02.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:38:02.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.197.253:37597
07:38:02.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.197.253
07:38:02.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:38:02.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/krb5kdc/kudu_127.4.197.253.keytab
07:38:02.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.197.253
07:38:02.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:38:02.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:38:02.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:38:02.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.197.212:38399
07:38:02.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:38:02.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:38:02.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:38:02.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:38:02.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.197.254:40799,127.4.197.253:37597,127.4.197.252:37301
07:38:02.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:38:02.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44963/jwks.json
07:38:02.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:38:02.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:38:02 dist-test-slave-tg41 krb5kdc[4900](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790235482, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.197.254@KRBTEST.COM for kudu/127.4.197.254@KRBTEST.COM
07:38:02.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:02.673372 4987 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.197.253" port: 37597 } has no permanent_uuid. Determining permanent_uuid...
07:38:02.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:02.675961 4938 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.197.253:37597: connect: Connection refused (error 111)
07:38:02.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:02.680810 4987 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.197.253:37597: Network error: Client connection negotiation failed: client connection to 127.4.197.253:37597: connect: Connection refused (error 111)
07:38:02.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:02.730353 4987 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.197.253" port: 37597 } attempt: 1
07:38:02.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:02.736937 4987 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.197.253:37597: Network error: Client connection negotiation failed: client connection to 127.4.197.253:37597: connect: Connection refused (error 111)
07:38:02.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:02.805423 4987 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.197.253" port: 37597 } attempt: 2
07:38:02.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:02.809809 4987 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.197.253:37597: Network error: Client connection negotiation failed: client connection to 127.4.197.253:37597: connect: Connection refused (error 111)
07:38:02.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:02.901274 4987 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.197.253" port: 37597 } attempt: 3
07:38:02.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:02.905866 4987 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.197.253:37597: Network error: Client connection negotiation failed: client connection to 127.4.197.253:37597: connect: Connection refused (error 111)
07:38:02.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:02.983589 4993 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:38:02.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:02.984113 4993 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:38:02.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:02.984694 4993 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:38:03.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:03.016716 4993 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:38:03.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:03.016932 4993 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:38:03.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:03.017194 4993 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:38:03.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:03.017385 4993 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:38:03.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:03.017570 4993 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:38:03.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:03.017838 4993 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:38:03.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:03.018010 4993 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44963/jwks.json
07:38:03.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:03.049782 4987 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.197.253" port: 37597 } attempt: 4
07:38:03.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:03.054558 4987 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.197.253:37597: Network error: Client connection negotiation failed: client connection to 127.4.197.253:37597: connect: Connection refused (error 111)
07:38:03.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:03.058571 4993 master_runner.cc:386] Master server non-default flags:
07:38:03.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:38:03.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.197.212:38399
07:38:03.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:38:03.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:38:03.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-1/data
07:38:03.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-1/wal
07:38:03.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:38:03.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:38:03.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.197.254:40799,127.4.197.253:37597,127.4.197.252:37301
07:38:03.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/krb5kdc/kudu_127.4.197.253.keytab
07:38:03.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.197.253
07:38:03.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:38:03.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:38:03.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:38:03.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.197.253:37597
07:38:03.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:38:03.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:38:03.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:38:03.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44963/jwks.json
07:38:03.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:38:03.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:38:03.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:38:03.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:38:03.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:38:03.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-1/data/info.pb
07:38:03.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.197.253
07:38:03.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:38:03.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:38:03.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:38:03.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:38:03.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:38:03.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:38:03.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-1/logs
07:38:03.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:38:03.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:38:03.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:38:03.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:38:03.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:38:03.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:38:03.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:38:03.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 06:42:32 UTC on ccycloud.quasar-lgarko.root.comops.site
07:38:03.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669406
07:38:03.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:38:03.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:03.061260 4993 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:38:03.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:03.062453 4993 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:38:03.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:03.084405 5001 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:38:03.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:03.353977 4987 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.197.253" port: 37597 } attempt: 5
07:38:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:03.359686 4987 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.197.253:37597: Network error: Client connection negotiation failed: client connection to 127.4.197.253:37597: connect: Connection refused (error 111)
07:38:03.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:03.907294 4987 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.197.253" port: 37597 } attempt: 6
07:38:03.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:03.917785 4987 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.197.253:37597: Network error: Client connection negotiation failed: client connection to 127.4.197.253:37597: connect: Connection refused (error 111)
07:38:04.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:03.086222 5003 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:38:04.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:03.084885 5000 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:38:04.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:04.394053 5002 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
07:38:04.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:04.394630 4993 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
07:38:04.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:04.400444 4993 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:38:04.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:04.403110 4993 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:38:04.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:04.404577 4993 hybrid_clock.cc:648] HybridClock initialized: now 1790235484404520 us; error 41 us; skew 500 ppm
07:38:04.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:04.405405 4993 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:38:04.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:38:04 dist-test-slave-tg41 krb5kdc[4900](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790235484, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.197.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:38:04.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:04.411680 4993 init.cc:377] Logged in from keytab as kudu/127.4.197.253@KRBTEST.COM (short username kudu)
07:38:04.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:04.417860 4993 webserver.cc:460] Webserver started at http://127.4.197.253:32881/ using document root <none> and password file <none>
07:38:04.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:04.418838 4993 fs_manager.cc:359] Metadata directory not provided
07:38:04.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:04.419076 4993 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:38:04.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:04.419574 4993 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:38:04.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:04.424377 4993 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-1/data/instance:
07:38:04.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a1f04fbbf283438c901b0bae2bcb6c45"
07:38:04.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:38:04 on dist-test-slave-tg41"
07:38:04.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:04.425655 4993 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-1/wal/instance:
07:38:04.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a1f04fbbf283438c901b0bae2bcb6c45"
07:38:04.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:38:04 on dist-test-slave-tg41"
07:38:04.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:04.434046 4993 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.006s sys 0.001s
07:38:04.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:04.440260 5015 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:38:04.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:04.441414 4993 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
07:38:04.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:04.441694 4993 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-1/data,/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-1/wal
07:38:04.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a1f04fbbf283438c901b0bae2bcb6c45"
07:38:04.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:38:04 on dist-test-slave-tg41"
07:38:04.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:04.442082 4993 fs_report.cc:383] FS layout report
07:38:04.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:38:04.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-1/wal
07:38:04.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-1/wal
07:38:04.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-1/data/data
07:38:04.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:38:04.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:38:04.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:38:04.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:38:04.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:38:04.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:38:04.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:38:04.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:38:04.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:38:04.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:38:04.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:38:04.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:04.495031 4993 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:38:04.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:04.501330 4993 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:38:04.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:04.501854 4993 kserver.cc:163] Server-wide thread pool size limit: 3276
07:38:04.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:04.592605 4993 rpc_server.cc:307] RPC server started. Bound to: 127.4.197.253:37597
07:38:04.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:04.592684 5066 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.197.253:37597 every 8 connection(s)
07:38:04.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:04.596880 4993 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-1/data/info.pb
07:38:04.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:04.603307 4887 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskDxb_lc/build/tsan/bin/kudu as pid 4993
07:38:04.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:04.603652 4887 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-1/wal/instance
07:38:04.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:04.603757 5067 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:38:04.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.4.197.252@KRBTEST.COM; defaulting to no policy
07:38:04.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:04.620007 5067 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.197.254" port: 40799 } has no permanent_uuid. Determining permanent_uuid...
07:38:04.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:38:04.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.4.197.252@KRBTEST.COM" created.
07:38:04.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:38:04 dist-test-slave-tg41 krb5kdc[4900](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790235484, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.197.253@KRBTEST.COM for kudu/127.4.197.254@KRBTEST.COM
07:38:04.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:38:04.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.197.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/krb5kdc/kudu_127.4.197.252.keytab.
07:38:04.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.197.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/krb5kdc/kudu_127.4.197.252.keytab.
07:38:04.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:04.659633 4887 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskDxb_lc/build/tsan/bin/kudu
07:38:04.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDxb_lc/build/tsan/bin/kudu
07:38:04.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-2/wal
07:38:04.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-2/data
07:38:04.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:38:04.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:38:04.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:38:04.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:38:04.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:38:04.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:38:04.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-2/logs
07:38:04.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-2/data/info.pb
07:38:04.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:38:04.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:38:04.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:38:04.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:38:04.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:38:04.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:38:04.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:38:04.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:38:04.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:38:04.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:38:04.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:38:04.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:38:04.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.197.252:37301
07:38:04.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.197.252
07:38:04.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:38:04.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/krb5kdc/kudu_127.4.197.252.keytab
07:38:04.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.197.252
07:38:04.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:38:04.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:38:04.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:38:04.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.197.212:38399
07:38:04.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:38:04.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:38:04.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:38:04.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:38:04.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.197.254:40799,127.4.197.253:37597,127.4.197.252:37301
07:38:04.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:38:04.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44963/jwks.json
07:38:04.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:38:04.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:04.663358 5067 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.197.253" port: 37597 } has no permanent_uuid. Determining permanent_uuid...
07:38:04.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:38:04 dist-test-slave-tg41 krb5kdc[4900](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790235484, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.197.253@KRBTEST.COM for kudu/127.4.197.253@KRBTEST.COM
07:38:04.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:04.694865 5067 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.197.252" port: 37301 } has no permanent_uuid. Determining permanent_uuid...
07:38:04.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:04.697245 5019 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.197.252:37301: connect: Connection refused (error 111)
07:38:04.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:04.701762 5067 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.197.252:37301: Network error: Client connection negotiation failed: client connection to 127.4.197.252:37301: connect: Connection refused (error 111)
07:38:04.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:04.751292 5067 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.197.252" port: 37301 } attempt: 1
07:38:04.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:04.755155 5067 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.197.252:37301: Network error: Client connection negotiation failed: client connection to 127.4.197.252:37301: connect: Connection refused (error 111)
07:38:04.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:04.823523 5067 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.197.252" port: 37301 } attempt: 2
07:38:04.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:04.827534 5067 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.197.252:37301: Network error: Client connection negotiation failed: client connection to 127.4.197.252:37301: connect: Connection refused (error 111)
07:38:04.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:04.918877 5067 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.197.252" port: 37301 } attempt: 3
07:38:04.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:04.922482 5067 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.197.252:37301: Network error: Client connection negotiation failed: client connection to 127.4.197.252:37301: connect: Connection refused (error 111)
07:38:04.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:04.978235 4987 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.197.253" port: 37597 } attempt: 7
07:38:04.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:04.991359 5073 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:38:04.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:04.992038 5073 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:38:04.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:38:04 dist-test-slave-tg41 krb5kdc[4900](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790235482, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.197.254@KRBTEST.COM for kudu/127.4.197.253@KRBTEST.COM
07:38:04.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:04.992831 5073 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:38:05.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:05.005362 4987 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.197.252" port: 37301 } has no permanent_uuid. Determining permanent_uuid...
07:38:05.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:05.011636 4987 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.197.252:37301: Network error: Client connection negotiation failed: client connection to 127.4.197.252:37301: connect: Connection refused (error 111)
07:38:05.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:05.027473 5073 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:38:05.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:05.027705 5073 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:38:05.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:05.027968 5073 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:38:05.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:05.028162 5073 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:38:05.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:05.028348 5073 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:38:05.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:05.028530 5073 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:38:05.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:05.028755 5073 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44963/jwks.json
07:38:05.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:05.065835 5067 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.197.252" port: 37301 } attempt: 4
07:38:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:05.069782 5067 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.197.252:37301: Network error: Client connection negotiation failed: client connection to 127.4.197.252:37301: connect: Connection refused (error 111)
07:38:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:05.069918 5073 master_runner.cc:386] Master server non-default flags:
07:38:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:38:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.197.212:38399
07:38:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:38:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:38:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-2/data
07:38:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-2/wal
07:38:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:38:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:38:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.197.254:40799,127.4.197.253:37597,127.4.197.252:37301
07:38:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/krb5kdc/kudu_127.4.197.252.keytab
07:38:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.197.252
07:38:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:38:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:38:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:38:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.197.252:37301
07:38:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:38:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:38:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:38:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44963/jwks.json
07:38:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:38:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:38:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:38:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:38:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:38:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-2/data/info.pb
07:38:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.197.252
07:38:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:38:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:38:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:38:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:38:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:38:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:38:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-2/logs
07:38:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:38:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:38:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:38:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:38:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:38:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:38:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:38:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 06:42:32 UTC on ccycloud.quasar-lgarko.root.comops.site
07:38:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669406
07:38:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:38:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:05.070045 4987 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.197.252" port: 37301 } attempt: 1
07:38:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:05.071425 5073 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:38:05.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:05.072578 5073 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:38:05.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:05.073444 4987 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.197.252:37301: Network error: Client connection negotiation failed: client connection to 127.4.197.252:37301: connect: Connection refused (error 111)
07:38:05.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:05.093187 5083 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:38:05.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:05.154917 4987 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.197.252" port: 37301 } attempt: 2
07:38:05.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:05.159600 4987 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.197.252:37301: Network error: Client connection negotiation failed: client connection to 127.4.197.252:37301: connect: Connection refused (error 111)
07:38:05.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:05.245174 4987 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.197.252" port: 37301 } attempt: 3
07:38:05.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:05.250730 4987 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.197.252:37301: Network error: Client connection negotiation failed: client connection to 127.4.197.252:37301: connect: Connection refused (error 111)
07:38:05.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:05.369304 5067 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.197.252" port: 37301 } attempt: 5
07:38:05.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:05.373677 5067 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.197.252:37301: Network error: Client connection negotiation failed: client connection to 127.4.197.252:37301: connect: Connection refused (error 111)
07:38:05.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:05.391311 4987 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.197.252" port: 37301 } attempt: 4
07:38:05.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:05.397176 4987 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.197.252:37301: Network error: Client connection negotiation failed: client connection to 127.4.197.252:37301: connect: Connection refused (error 111)
07:38:05.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:05.680769 4987 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.197.252" port: 37301 } attempt: 5
07:38:05.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:05.686647 4987 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.197.252:37301: Network error: Client connection negotiation failed: client connection to 127.4.197.252:37301: connect: Connection refused (error 111)
07:38:05.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:05.921104 5067 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.197.252" port: 37301 } attempt: 6
07:38:05.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:05.927814 5067 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.197.252:37301: Network error: Client connection negotiation failed: client connection to 127.4.197.252:37301: connect: Connection refused (error 111)
07:38:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.239115 4987 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.197.252" port: 37301 } attempt: 6
07:38:06.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:06.255779 4987 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.197.252:37301: Network error: Client connection negotiation failed: client connection to 127.4.197.252:37301: connect: Connection refused (error 111)
07:38:06.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:05.093236 5084 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:38:06.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:05.095484 5073 server_base.cc:1034] running on GCE node
07:38:06.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:05.093660 5086 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:38:06.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.375432 5073 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:38:06.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.378696 5073 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:38:06.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.380136 5073 hybrid_clock.cc:648] HybridClock initialized: now 1790235486380096 us; error 66 us; skew 500 ppm
07:38:06.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.380898 5073 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:38:06.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:38:06 dist-test-slave-tg41 krb5kdc[4900](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790235486, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.197.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:38:06.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.387598 5073 init.cc:377] Logged in from keytab as kudu/127.4.197.252@KRBTEST.COM (short username kudu)
07:38:06.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.393131 5073 webserver.cc:460] Webserver started at http://127.4.197.252:38613/ using document root <none> and password file <none>
07:38:06.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.394050 5073 fs_manager.cc:359] Metadata directory not provided
07:38:06.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.394235 5073 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:38:06.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.394621 5073 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:38:06.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.399402 5073 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-2/data/instance:
07:38:06.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "824f0916aaf84abeab55bd0ff85a9b64"
07:38:06.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:38:06 on dist-test-slave-tg41"
07:38:06.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.400403 5073 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-2/wal/instance:
07:38:06.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "824f0916aaf84abeab55bd0ff85a9b64"
07:38:06.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:38:06 on dist-test-slave-tg41"
07:38:06.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.408783 5073 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.005s sys 0.005s
07:38:06.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.414954 5100 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:38:06.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.416116 5073 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
07:38:06.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.416401 5073 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-2/data,/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-2/wal
07:38:06.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "824f0916aaf84abeab55bd0ff85a9b64"
07:38:06.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:38:06 on dist-test-slave-tg41"
07:38:06.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.416706 5073 fs_report.cc:383] FS layout report
07:38:06.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:38:06.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-2/wal
07:38:06.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-2/wal
07:38:06.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-2/data/data
07:38:06.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:38:06.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:38:06.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:38:06.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:38:06.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:38:06.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:38:06.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:38:06.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:38:06.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:38:06.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:38:06.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:38:06.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.457229 5073 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:38:06.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.464017 5073 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:38:06.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.464459 5073 kserver.cc:163] Server-wide thread pool size limit: 3276
07:38:06.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.546891 5073 rpc_server.cc:307] RPC server started. Bound to: 127.4.197.252:37301
07:38:06.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.546975 5151 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.197.252:37301 every 8 connection(s)
07:38:06.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.549827 5073 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-2/data/info.pb
07:38:06.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.556077 5152 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:38:06.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.558291 4887 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskDxb_lc/build/tsan/bin/kudu as pid 5073
07:38:06.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.559353 4887 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-2/wal/instance
07:38:06.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.578292 5152 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.197.254" port: 40799 } has no permanent_uuid. Determining permanent_uuid...
07:38:06.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:38:06 dist-test-slave-tg41 krb5kdc[4900](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790235486, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.197.252@KRBTEST.COM for kudu/127.4.197.254@KRBTEST.COM
07:38:06.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:38:06 dist-test-slave-tg41 krb5kdc[4900](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790235479, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.4.197.252@KRBTEST.COM
07:38:06.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.621807 5152 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.197.253" port: 37597 } has no permanent_uuid. Determining permanent_uuid...
07:38:06.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:38:06 dist-test-slave-tg41 krb5kdc[4900](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790235486, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.197.252@KRBTEST.COM for kudu/127.4.197.253@KRBTEST.COM
07:38:06.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:38:06 dist-test-slave-tg41 krb5kdc[4900](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790235479, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.4.197.253@KRBTEST.COM
07:38:06.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.650446 5152 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.197.252" port: 37301 } has no permanent_uuid. Determining permanent_uuid...
07:38:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:38:06 dist-test-slave-tg41 krb5kdc[4900](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790235486, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.197.252@KRBTEST.COM for kudu/127.4.197.252@KRBTEST.COM
07:38:06.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:38:06 dist-test-slave-tg41 krb5kdc[4900](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790235479, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.4.197.254@KRBTEST.COM
07:38:06.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.682363 5152 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64: Bootstrap starting.
07:38:06.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.689320 5152 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64: Neither blocks nor log segments found. Creating new log.
07:38:06.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.691045 4887 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
07:38:06.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.691947 5152 log.cc:824] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64: Log is configured to *not* fsync() on all Append() calls
07:38:06.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.697253 5152 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64: No bootstrap required, opened a new log
07:38:06.703 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:38:06.704 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
07:38:06.718 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
07:38:06.719 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
07:38:06.725 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.4.197.254"
port: 40799
}
bound_http_address {
host: "127.4.197.254"
port: 39931
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.4.197.253"
port: 37597
}
bound_http_address {
host: "127.4.197.253"
port: 32881
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.4.197.252"
port: 37301
}
bound_http_address {
host: "127.4.197.252"
port: 38613
}
}
}
07:38:06.725 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
07:38:06.726 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:38:06.726 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
07:38:06.728 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
07:38:06.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.730592 5152 raft_consensus.cc:348] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "925836e479bc498db068663e717e7b91" member_type: VOTER last_known_addr { host: "127.4.197.254" port: 40799 } } peers { permanent_uuid: "a1f04fbbf283438c901b0bae2bcb6c45" member_type: VOTER last_known_addr { host: "127.4.197.253" port: 37597 } } peers { permanent_uuid: "824f0916aaf84abeab55bd0ff85a9b64" member_type: VOTER last_known_addr { host: "127.4.197.252" port: 37301 } }
07:38:06.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.731528 5152 raft_consensus.cc:374] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:38:06.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.732016 5152 raft_consensus.cc:729] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 824f0916aaf84abeab55bd0ff85a9b64, State: Initialized, Role: FOLLOWER
07:38:06.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.733803 5152 consensus_queue.cc:260] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "925836e479bc498db068663e717e7b91" member_type: VOTER last_known_addr { host: "127.4.197.254" port: 40799 } } peers { permanent_uuid: "a1f04fbbf283438c901b0bae2bcb6c45" member_type: VOTER last_known_addr { host: "127.4.197.253" port: 37597 } } peers { permanent_uuid: "824f0916aaf84abeab55bd0ff85a9b64" member_type: VOTER last_known_addr { host: "127.4.197.252" port: 37301 } }
07:38:06.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.738425 5161 sys_catalog.cc:455] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "925836e479bc498db068663e717e7b91" member_type: VOTER last_known_addr { host: "127.4.197.254" port: 40799 } } peers { permanent_uuid: "a1f04fbbf283438c901b0bae2bcb6c45" member_type: VOTER last_known_addr { host: "127.4.197.253" port: 37597 } } peers { permanent_uuid: "824f0916aaf84abeab55bd0ff85a9b64" member_type: VOTER last_known_addr { host: "127.4.197.252" port: 37301 } } }
07:38:06.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.739502 5161 sys_catalog.cc:458] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [sys.catalog]: This master's current role is: FOLLOWER
07:38:06.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.739916 5152 sys_catalog.cc:564] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [sys.catalog]: configured and running, proceeding with master startup.
07:38:06.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:38:06 dist-test-slave-tg41 krb5kdc[4900](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790235486, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:38:06.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
07:38:06.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:06.772768 5174 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:38:06.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:06.773118 5174 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:38:06.775 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:38:06.777 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/krb5kdc/krb5cc
07:38:06.782 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
07:38:06.783 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/krb5kdc/krb5cc
07:38:06.785 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
07:38:06.789 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
07:38:06.791 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
07:38:06.791 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
07:38:06.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.769057 5073 master_runner.cc:186] Error getting master registration for 127.4.197.254:40799: OK, instance_id { permanent_uuid: "925836e479bc498db068663e717e7b91" instance_seqno: 1790235482550183 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
07:38:06.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.804466 5073 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:38:06.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:38:06 dist-test-slave-tg41 krb5kdc[4900](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790235486, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:38:06.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
07:38:06.825 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:38:06.826 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/krb5kdc/krb5cc
07:38:06.827 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
07:38:06.828 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.4.197.253:37597
07:38:06.829 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
07:38:06.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.830462 4887 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskDxb_lc/build/tsan/bin/kudu with pid 4993
07:38:06.851 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:38:06.852 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.4.197.254:40799
07:38:06.852 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
07:38:06.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.852807 4887 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskDxb_lc/build/tsan/bin/kudu with pid 4915
07:38:06.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.859163 5161 raft_consensus.cc:482] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:38:06.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.859710 5161 raft_consensus.cc:504] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "925836e479bc498db068663e717e7b91" member_type: VOTER last_known_addr { host: "127.4.197.254" port: 40799 } } peers { permanent_uuid: "a1f04fbbf283438c901b0bae2bcb6c45" member_type: VOTER last_known_addr { host: "127.4.197.253" port: 37597 } } peers { permanent_uuid: "824f0916aaf84abeab55bd0ff85a9b64" member_type: VOTER last_known_addr { host: "127.4.197.252" port: 37301 } }
07:38:06.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.862323 5161 leader_election.cc:288] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 925836e479bc498db068663e717e7b91 (127.4.197.254:40799), a1f04fbbf283438c901b0bae2bcb6c45 (127.4.197.253:37597)
07:38:06.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:06.863925 5103 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.197.253:37597: connect: Connection refused (error 111)
07:38:06.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:06.869024 5103 leader_election.cc:334] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a1f04fbbf283438c901b0bae2bcb6c45 (127.4.197.253:37597): Network error: Client connection negotiation failed: client connection to 127.4.197.253:37597: connect: Connection refused (error 111)
07:38:06.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:06.883131 5103 connection.cc:535] client connection to 127.4.197.254:40799 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
07:38:06.884 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:38:06.884 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.4.197.252:37301
07:38:06.884 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
07:38:06.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:06.885874 5103 leader_election.cc:334] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 925836e479bc498db068663e717e7b91 (127.4.197.254:40799): Network error: Client connection negotiation failed: client connection to 127.4.197.254:40799: connect: Connection refused (error 111)
07:38:06.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.886459 5103 leader_election.cc:302] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 824f0916aaf84abeab55bd0ff85a9b64; no voters: 925836e479bc498db068663e717e7b91, a1f04fbbf283438c901b0bae2bcb6c45
07:38:06.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.887390 5161 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:38:06.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.887830 4887 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskDxb_lc/build/tsan/bin/kudu with pid 5073
07:38:06.908 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:38:06.908 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.4.197.253:37597
07:38:06.910 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
07:38:06.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:06.912083 4887 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskDxb_lc/build/tsan/bin/kudu
07:38:06.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDxb_lc/build/tsan/bin/kudu
07:38:06.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-1/wal
07:38:06.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-1/data
07:38:06.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:38:06.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:38:06.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:38:06.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:38:06.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:38:06.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:38:06.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-1/logs
07:38:06.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-1/data/info.pb
07:38:06.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:38:06.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:38:06.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:38:06.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:38:06.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:38:06.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:38:06.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:38:06.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:38:06.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:38:06.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:38:06.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:38:06.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:38:06.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.197.253:37597
07:38:06.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.197.253
07:38:06.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32881
07:38:06.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/krb5kdc/kudu_127.4.197.253.keytab
07:38:06.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.197.253
07:38:06.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:38:06.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:38:06.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:38:06.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.197.212:38399
07:38:06.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:38:06.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:38:06.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:38:06.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:38:06.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.197.254:40799,127.4.197.253:37597,127.4.197.252:37301
07:38:06.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:38:06.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44963/jwks.json
07:38:06.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:38:07.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:07.222909 5179 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:38:07.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:07.223426 5179 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:38:07.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:07.224001 5179 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:38:07.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:07.257091 5179 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:38:07.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:07.257405 5179 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:38:07.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:07.257885 5179 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:38:07.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:07.258244 5179 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:38:07.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:07.258563 5179 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:38:07.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:07.258765 5179 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:38:07.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:07.258908 5179 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44963/jwks.json
07:38:07.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:07.297080 5179 master_runner.cc:386] Master server non-default flags:
07:38:07.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:38:07.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.197.212:38399
07:38:07.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:38:07.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:38:07.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-1/data
07:38:07.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-1/wal
07:38:07.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:38:07.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:38:07.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.197.254:40799,127.4.197.253:37597,127.4.197.252:37301
07:38:07.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/krb5kdc/kudu_127.4.197.253.keytab
07:38:07.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.197.253
07:38:07.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:38:07.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:38:07.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:38:07.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.197.253:37597
07:38:07.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:38:07.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:38:07.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:38:07.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44963/jwks.json
07:38:07.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:38:07.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:38:07.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:38:07.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:38:07.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:38:07.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-1/data/info.pb
07:38:07.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.197.253
07:38:07.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32881
07:38:07.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:38:07.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:38:07.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:38:07.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:38:07.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:38:07.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-1/logs
07:38:07.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:38:07.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:38:07.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:38:07.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:38:07.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:38:07.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:38:07.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:38:07.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 06:42:32 UTC on ccycloud.quasar-lgarko.root.comops.site
07:38:07.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669406
07:38:07.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:38:07.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:07.298370 5179 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:38:07.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:07.299316 5179 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:38:07.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:07.318400 5186 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:38:08.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:07.318681 5185 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:38:08.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:07.319697 5179 server_base.cc:1034] running on GCE node
07:38:08.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:07.318440 5188 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:38:08.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:08.546157 5179 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:38:08.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:08.549299 5179 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:38:08.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:08.550683 5179 hybrid_clock.cc:648] HybridClock initialized: now 1790235488550655 us; error 40 us; skew 500 ppm
07:38:08.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:08.551719 5179 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:38:08.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:38:08 dist-test-slave-tg41 krb5kdc[4900](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790235488, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.197.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:38:08.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:08.565732 5179 init.cc:377] Logged in from keytab as kudu/127.4.197.253@KRBTEST.COM (short username kudu)
07:38:08.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:08.571305 5179 webserver.cc:460] Webserver started at http://127.4.197.253:32881/ using document root <none> and password file <none>
07:38:08.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:08.572461 5179 fs_manager.cc:359] Metadata directory not provided
07:38:08.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:08.572722 5179 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:38:08.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:08.582581 5179 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.007s sys 0.000s
07:38:08.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:08.586675 5196 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:38:08.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:08.587697 5179 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.001s
07:38:08.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:08.588090 5179 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-1/data,/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-1/wal
07:38:08.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a1f04fbbf283438c901b0bae2bcb6c45"
07:38:08.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:38:04 on dist-test-slave-tg41"
07:38:08.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:08.590555 5179 fs_report.cc:383] FS layout report
07:38:08.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:38:08.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-1/wal
07:38:08.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-1/wal
07:38:08.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-1/data/data
07:38:08.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:38:08.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:38:08.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:38:08.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:38:08.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:38:08.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:38:08.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:38:08.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:38:08.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:38:08.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:38:08.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:38:08.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:08.630512 5179 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:38:08.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:08.636951 5179 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:38:08.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:08.637507 5179 kserver.cc:163] Server-wide thread pool size limit: 3276
07:38:08.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:08.713398 5179 rpc_server.cc:307] RPC server started. Bound to: 127.4.197.253:37597
07:38:08.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:08.713518 5247 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.197.253:37597 every 8 connection(s)
07:38:08.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:08.716353 5179 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-1/data/info.pb
07:38:08.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:08.722520 4887 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskDxb_lc/build/tsan/bin/kudu as pid 5179
07:38:08.723 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:38:08.723 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.4.197.254:40799
07:38:08.723 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
07:38:08.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:08.725309 4887 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskDxb_lc/build/tsan/bin/kudu
07:38:08.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDxb_lc/build/tsan/bin/kudu
07:38:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-0/wal
07:38:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-0/data
07:38:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:38:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:38:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:38:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:38:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:38:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:38:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-0/logs
07:38:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-0/data/info.pb
07:38:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:38:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:38:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:38:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:38:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:38:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:38:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:38:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:38:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:38:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:38:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:38:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:38:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.197.254:40799
07:38:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.197.254
07:38:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39931
07:38:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/krb5kdc/kudu_127.4.197.254.keytab
07:38:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.197.254
07:38:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:38:08.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:38:08.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:38:08.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.197.212:38399
07:38:08.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:38:08.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:38:08.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:38:08.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:38:08.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.197.254:40799,127.4.197.253:37597,127.4.197.252:37301
07:38:08.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:38:08.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44963/jwks.json
07:38:08.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:38:08.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:08.729930 5248 sys_catalog.cc:263] Verifying existing consensus state
07:38:08.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260924 07:38:08.731055 5248 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
07:38:08.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:08.731467 5179 master.cc:537] Master@127.4.197.253:37597 shutting down...
07:38:08.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:08.791967 5179 master.cc:552] Master@127.4.197.253:37597 shutdown complete.
07:38:08.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
07:38:09.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:09.054343 5250 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:38:09.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:09.054874 5250 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:38:09.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:09.055430 5250 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:38:09.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:09.086795 5250 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:38:09.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:09.087033 5250 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:38:09.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:09.087292 5250 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:38:09.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:09.087491 5250 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:38:09.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:09.087674 5250 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:38:09.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:09.087859 5250 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:38:09.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:09.087996 5250 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44963/jwks.json
07:38:09.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:09.125367 5250 master_runner.cc:386] Master server non-default flags:
07:38:09.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:38:09.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.197.212:38399
07:38:09.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:38:09.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:38:09.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-0/data
07:38:09.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-0/wal
07:38:09.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:38:09.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:38:09.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.197.254:40799,127.4.197.253:37597,127.4.197.252:37301
07:38:09.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/krb5kdc/kudu_127.4.197.254.keytab
07:38:09.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.197.254
07:38:09.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:38:09.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:38:09.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:38:09.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.197.254:40799
07:38:09.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:38:09.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:38:09.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:38:09.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44963/jwks.json
07:38:09.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:38:09.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:38:09.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:38:09.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:38:09.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:38:09.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-0/data/info.pb
07:38:09.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.197.254
07:38:09.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39931
07:38:09.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:38:09.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:38:09.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:38:09.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:38:09.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:38:09.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-0/logs
07:38:09.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:38:09.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:38:09.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:38:09.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:38:09.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:38:09.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:38:09.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:38:09.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 06:42:32 UTC on ccycloud.quasar-lgarko.root.comops.site
07:38:09.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669406
07:38:09.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:38:09.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:09.126652 5250 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:38:09.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:09.127864 5250 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:38:09.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:09.146008 5257 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:38:10.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:09.146674 5260 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:38:10.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:09.146399 5258 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:38:10.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:10.409379 5259 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
07:38:10.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:10.409601 5250 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
07:38:10.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:10.413852 5250 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:38:10.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:10.416867 5250 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:38:10.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:10.418310 5250 hybrid_clock.cc:648] HybridClock initialized: now 1790235490418198 us; error 107 us; skew 500 ppm
07:38:10.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:10.419071 5250 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:38:10.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:38:10 dist-test-slave-tg41 krb5kdc[4900](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790235490, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.197.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:38:10.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:10.424927 5250 init.cc:377] Logged in from keytab as kudu/127.4.197.254@KRBTEST.COM (short username kudu)
07:38:10.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:10.430328 5250 webserver.cc:460] Webserver started at http://127.4.197.254:39931/ using document root <none> and password file <none>
07:38:10.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:10.431205 5250 fs_manager.cc:359] Metadata directory not provided
07:38:10.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:10.431393 5250 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:38:10.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:10.439311 5250 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.000s sys 0.004s
07:38:10.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:10.443528 5268 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:38:10.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:10.444664 5250 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
07:38:10.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:10.444943 5250 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-0/data,/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-0/wal
07:38:10.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "925836e479bc498db068663e717e7b91"
07:38:10.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:38:02 on dist-test-slave-tg41"
07:38:10.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:10.447420 5250 fs_report.cc:383] FS layout report
07:38:10.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:38:10.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-0/wal
07:38:10.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-0/wal
07:38:10.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-0/data/data
07:38:10.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:38:10.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:38:10.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:38:10.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:38:10.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:38:10.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:38:10.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:38:10.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:38:10.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:38:10.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:38:10.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:38:10.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:10.491951 5250 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:38:10.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:10.497867 5250 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:38:10.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:10.498283 5250 kserver.cc:163] Server-wide thread pool size limit: 3276
07:38:10.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:10.574266 5250 rpc_server.cc:307] RPC server started. Bound to: 127.4.197.254:40799
07:38:10.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:10.574391 5319 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.197.254:40799 every 8 connection(s)
07:38:10.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:10.577101 5250 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-0/data/info.pb
07:38:10.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:10.582626 4887 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskDxb_lc/build/tsan/bin/kudu as pid 5250
07:38:10.583 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:38:10.583 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.4.197.252:37301
07:38:10.584 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
07:38:10.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:10.585254 4887 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskDxb_lc/build/tsan/bin/kudu
07:38:10.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDxb_lc/build/tsan/bin/kudu
07:38:10.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-2/wal
07:38:10.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-2/data
07:38:10.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:38:10.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:38:10.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:38:10.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:38:10.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:38:10.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:38:10.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-2/logs
07:38:10.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-2/data/info.pb
07:38:10.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:38:10.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:38:10.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:38:10.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:38:10.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:38:10.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:38:10.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:38:10.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:38:10.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:38:10.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:38:10.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:38:10.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:38:10.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.197.252:37301
07:38:10.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.197.252
07:38:10.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38613
07:38:10.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/krb5kdc/kudu_127.4.197.252.keytab
07:38:10.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.197.252
07:38:10.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:38:10.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:38:10.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:38:10.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.197.212:38399
07:38:10.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:38:10.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:38:10.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:38:10.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:38:10.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.197.254:40799,127.4.197.253:37597,127.4.197.252:37301
07:38:10.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:38:10.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44963/jwks.json
07:38:10.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:38:10.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:10.588191 5320 sys_catalog.cc:263] Verifying existing consensus state
07:38:10.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260924 07:38:10.588850 5320 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
07:38:10.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:10.589210 5250 master.cc:537] Master@127.4.197.254:40799 shutting down...
07:38:10.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:10.658550 5250 master.cc:552] Master@127.4.197.254:40799 shutdown complete.
07:38:10.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
07:38:10.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:10.902701 5322 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:38:10.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:10.903450 5322 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:38:10.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:10.904170 5322 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:38:10.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:10.939083 5322 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:38:10.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:10.939364 5322 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:38:10.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:10.939752 5322 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:38:10.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:10.940109 5322 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:38:10.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:10.940416 5322 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:38:10.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:10.940719 5322 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:38:10.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:10.940945 5322 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44963/jwks.json
07:38:10.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:10.978610 5322 master_runner.cc:386] Master server non-default flags:
07:38:10.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:38:10.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.197.212:38399
07:38:10.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:38:10.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:38:10.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-2/data
07:38:10.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-2/wal
07:38:10.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:38:10.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:38:10.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.197.254:40799,127.4.197.253:37597,127.4.197.252:37301
07:38:10.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/krb5kdc/kudu_127.4.197.252.keytab
07:38:10.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.197.252
07:38:10.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:38:10.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:38:10.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:38:10.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.197.252:37301
07:38:10.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:38:10.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:38:10.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:38:10.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44963/jwks.json
07:38:10.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:38:10.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:38:10.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:38:10.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:38:10.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:38:10.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-2/data/info.pb
07:38:10.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.197.252
07:38:10.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38613
07:38:10.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:38:10.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:38:10.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:38:10.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:38:10.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:38:10.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-2/logs
07:38:10.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:38:10.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:38:10.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:38:10.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:38:10.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:38:10.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:38:10.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:38:10.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 06:42:32 UTC on ccycloud.quasar-lgarko.root.comops.site
07:38:10.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669406
07:38:10.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:38:10.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:10.980134 5322 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:38:10.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:10.981503 5322 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:38:11.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:10.999879 5329 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:38:12.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:11.002033 5331 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:38:12.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:11.002975 5322 server_base.cc:1034] running on GCE node
07:38:12.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:10.999987 5328 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:38:12.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:12.243608 5322 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:38:12.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:12.246632 5322 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:38:12.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:12.248287 5322 hybrid_clock.cc:648] HybridClock initialized: now 1790235492248215 us; error 92 us; skew 500 ppm
07:38:12.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:12.249107 5322 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:38:12.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:38:12 dist-test-slave-tg41 krb5kdc[4900](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790235492, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.197.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:38:12.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:12.255004 5322 init.cc:377] Logged in from keytab as kudu/127.4.197.252@KRBTEST.COM (short username kudu)
07:38:12.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:12.260076 5322 webserver.cc:460] Webserver started at http://127.4.197.252:38613/ using document root <none> and password file <none>
07:38:12.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:12.260896 5322 fs_manager.cc:359] Metadata directory not provided
07:38:12.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:12.261086 5322 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:38:12.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:12.269450 5322 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.006s sys 0.000s
07:38:12.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:12.275285 5341 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:38:12.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:12.276429 5322 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.002s
07:38:12.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:12.276743 5322 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-2/data,/tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-2/wal
07:38:12.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "824f0916aaf84abeab55bd0ff85a9b64"
07:38:12.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:38:06 on dist-test-slave-tg41"
07:38:12.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:12.279139 5322 fs_report.cc:383] FS layout report
07:38:12.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:38:12.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-2/wal
07:38:12.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-2/wal
07:38:12.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-2/data/data
07:38:12.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:38:12.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:38:12.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:38:12.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:38:12.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:38:12.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:38:12.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:38:12.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:38:12.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:38:12.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:38:12.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:38:12.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:12.350536 5322 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:38:12.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:12.356463 5322 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:38:12.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:12.356887 5322 kserver.cc:163] Server-wide thread pool size limit: 3276
07:38:12.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:12.430572 5322 rpc_server.cc:307] RPC server started. Bound to: 127.4.197.252:37301
07:38:12.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:12.430660 5393 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.197.252:37301 every 8 connection(s)
07:38:12.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:12.433899 5322 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskDxb_lc/test-tmp/mini-kudu-cluster4749702899361548452/master-2/data/info.pb
07:38:12.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:12.441418 4887 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskDxb_lc/build/tsan/bin/kudu as pid 5322
07:38:12.442 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:38:12.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:12.445272 5394 sys_catalog.cc:263] Verifying existing consensus state
07:38:12.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:12.450850 5394 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64: Bootstrap starting.
07:38:12.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:12.486690 5394 log.cc:824] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64: Log is configured to *not* fsync() on all Append() calls
07:38:12.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:12.491619 5394 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
07:38:12.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:12.492321 5394 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64: Bootstrap complete.
07:38:12.493 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.4.197.253:37597(127.4.197.253:37597)] connecting to peer
07:38:12.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:12.516953 5394 raft_consensus.cc:348] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "925836e479bc498db068663e717e7b91" member_type: VOTER last_known_addr { host: "127.4.197.254" port: 40799 } } peers { permanent_uuid: "a1f04fbbf283438c901b0bae2bcb6c45" member_type: VOTER last_known_addr { host: "127.4.197.253" port: 37597 } } peers { permanent_uuid: "824f0916aaf84abeab55bd0ff85a9b64" member_type: VOTER last_known_addr { host: "127.4.197.252" port: 37301 } }
07:38:12.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:12.519688 5394 raft_consensus.cc:374] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:38:12.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:12.520033 5394 raft_consensus.cc:729] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 824f0916aaf84abeab55bd0ff85a9b64, State: Initialized, Role: FOLLOWER
07:38:12.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:12.520860 5394 consensus_queue.cc:260] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "925836e479bc498db068663e717e7b91" member_type: VOTER last_known_addr { host: "127.4.197.254" port: 40799 } } peers { permanent_uuid: "a1f04fbbf283438c901b0bae2bcb6c45" member_type: VOTER last_known_addr { host: "127.4.197.253" port: 37597 } } peers { permanent_uuid: "824f0916aaf84abeab55bd0ff85a9b64" member_type: VOTER last_known_addr { host: "127.4.197.252" port: 37301 } }
07:38:12.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:12.523746 5398 sys_catalog.cc:455] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "925836e479bc498db068663e717e7b91" member_type: VOTER last_known_addr { host: "127.4.197.254" port: 40799 } } peers { permanent_uuid: "a1f04fbbf283438c901b0bae2bcb6c45" member_type: VOTER last_known_addr { host: "127.4.197.253" port: 37597 } } peers { permanent_uuid: "824f0916aaf84abeab55bd0ff85a9b64" member_type: VOTER last_known_addr { host: "127.4.197.252" port: 37301 } } }
07:38:12.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:12.524755 5398 sys_catalog.cc:458] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [sys.catalog]: This master's current role is: FOLLOWER
07:38:12.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:12.525629 5394 sys_catalog.cc:564] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [sys.catalog]: configured and running, proceeding with master startup.
07:38:12.556 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.4.197.254:40799(127.4.197.254:40799)] connecting to peer
07:38:12.558 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.4.197.252:37301(127.4.197.252:37301)] connecting to peer
07:38:12.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:12.587594 5411 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:38:12.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:12.588160 5411 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:38:12.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:12.612278 5348 tablet.cc:2369] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64: Can't schedule compaction. Clean time has not been advanced past its initial value.
07:38:12.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:12.604909 5322 master_runner.cc:186] Error getting master registration for 127.4.197.254:40799: Network error: Client connection negotiation failed: client connection to 127.4.197.254:40799: connect: Connection refused (error 111),
07:38:12.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:12.640749 5322 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:38:12.641 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:12.642 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:12.642 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.197.253:37597(127.4.197.253:37597)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:12.644 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.197.254:40799(127.4.197.254:40799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:12.644 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.253:37597: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:12.646 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.254:40799: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:12.646 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.4.197.252:37301(127.4.197.252:37301)] Successfully connected to peer
07:38:12.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:12.655583 5398 raft_consensus.cc:482] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:38:12.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:12.656144 5398 raft_consensus.cc:504] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "925836e479bc498db068663e717e7b91" member_type: VOTER last_known_addr { host: "127.4.197.254" port: 40799 } } peers { permanent_uuid: "a1f04fbbf283438c901b0bae2bcb6c45" member_type: VOTER last_known_addr { host: "127.4.197.253" port: 37597 } } peers { permanent_uuid: "824f0916aaf84abeab55bd0ff85a9b64" member_type: VOTER last_known_addr { host: "127.4.197.252" port: 37301 } }
07:38:12.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:12.659317 5398 leader_election.cc:288] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 925836e479bc498db068663e717e7b91 (127.4.197.254:40799), a1f04fbbf283438c901b0bae2bcb6c45 (127.4.197.253:37597)
07:38:12.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:12.663514 5344 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.197.254:40799: connect: Connection refused (error 111)
07:38:12.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:12.668731 5344 leader_election.cc:334] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 925836e479bc498db068663e717e7b91 (127.4.197.254:40799): Network error: Client connection negotiation failed: client connection to 127.4.197.254:40799: connect: Connection refused (error 111)
07:38:12.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:12.669247 5344 leader_election.cc:334] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a1f04fbbf283438c901b0bae2bcb6c45 (127.4.197.253:37597): Network error: Client connection negotiation failed: client connection to 127.4.197.253:37597: connect: Connection refused (error 111)
07:38:12.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:12.669575 5344 leader_election.cc:302] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 824f0916aaf84abeab55bd0ff85a9b64; no voters: 925836e479bc498db068663e717e7b91, a1f04fbbf283438c901b0bae2bcb6c45
07:38:12.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:12.670370 5398 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:38:12.737 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
07:38:12.749 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.4.197.252
07:38:12.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:38:12 dist-test-slave-tg41 krb5kdc[4900](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790235486, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.4.197.252@KRBTEST.COM
07:38:13.006 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xdc30b1af, L:/127.0.0.1:39792 - R:/127.4.197.252:37301] using SASL/GSSAPI
07:38:13.026 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301; will retry
07:38:13.048 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.253:37597(127.4.197.253:37597)] connecting to peer
07:38:13.050 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.254:40799(127.4.197.254:40799)] connecting to peer
07:38:13.054 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:13.054 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.197.253:37597(127.4.197.253:37597)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:13.055 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.253:37597: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:13.057 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:13.057 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.197.254:40799(127.4.197.254:40799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:13.059 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.254:40799: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:13.061 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301; will retry
07:38:13.066 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.253:37597(127.4.197.253:37597)] connecting to peer
07:38:13.068 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.254:40799(127.4.197.254:40799)] connecting to peer
07:38:13.068 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:13.068 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.197.253:37597(127.4.197.253:37597)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:13.069 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.253:37597: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:13.071 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:13.072 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.197.254:40799(127.4.197.254:40799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:13.073 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.254:40799: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:13.076 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301; will retry
07:38:13.086 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.253:37597(127.4.197.253:37597)] connecting to peer
07:38:13.087 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.254:40799(127.4.197.254:40799)] connecting to peer
07:38:13.088 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:13.088 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.197.253:37597(127.4.197.253:37597)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:13.091 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.253:37597: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:13.094 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:13.094 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.197.254:40799(127.4.197.254:40799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:13.095 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.254:40799: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:13.098 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301; will retry
07:38:13.127 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.253:37597(127.4.197.253:37597)] connecting to peer
07:38:13.128 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.254:40799(127.4.197.254:40799)] connecting to peer
07:38:13.130 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:13.130 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.197.253:37597(127.4.197.253:37597)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:13.131 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.253:37597: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:13.133 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:13.134 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.197.254:40799(127.4.197.254:40799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:13.134 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.254:40799: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:13.135 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301; will retry
07:38:13.167 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.253:37597(127.4.197.253:37597)] connecting to peer
07:38:13.168 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.254:40799(127.4.197.254:40799)] connecting to peer
07:38:13.169 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:13.170 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.197.253:37597(127.4.197.253:37597)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:13.171 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.253:37597: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:13.186 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:13.186 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.197.254:40799(127.4.197.254:40799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:13.187 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.254:40799: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:13.188 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301; will retry
07:38:13.267 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.253:37597(127.4.197.253:37597)] connecting to peer
07:38:13.269 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.254:40799(127.4.197.254:40799)] connecting to peer
07:38:13.269 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:13.269 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.197.253:37597(127.4.197.253:37597)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:13.270 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.253:37597: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:13.275 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:13.276 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.197.254:40799(127.4.197.254:40799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:13.278 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.254:40799: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:13.279 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301; will retry
07:38:13.327 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.253:37597(127.4.197.253:37597)] connecting to peer
07:38:13.329 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.254:40799(127.4.197.254:40799)] connecting to peer
07:38:13.330 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:13.330 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.197.253:37597(127.4.197.253:37597)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:13.330 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.253:37597: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:13.333 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:13.333 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.197.254:40799(127.4.197.254:40799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:13.334 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.254:40799: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:13.336 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301; will retry
07:38:13.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:13.434631 5390 messenger.cc:188] Invalid argument: socket address is ambiguous: no data on pending connections for acceptor pool at 127.4.197.252:37301
07:38:13.466 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.253:37597(127.4.197.253:37597)] connecting to peer
07:38:13.468 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.254:40799(127.4.197.254:40799)] connecting to peer
07:38:13.468 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:13.468 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.197.253:37597(127.4.197.253:37597)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:13.469 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.253:37597: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:13.471 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:13.471 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.197.254:40799(127.4.197.254:40799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:13.472 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.254:40799: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:13.476 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301; will retry
07:38:13.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:13.592561 5411 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:38:13.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:13.592890 5411 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:38:13.746 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.253:37597(127.4.197.253:37597)] connecting to peer
07:38:13.747 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.254:40799(127.4.197.254:40799)] connecting to peer
07:38:13.749 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:13.749 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.197.253:37597(127.4.197.253:37597)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:13.754 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.253:37597: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:13.755 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:13.755 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.197.254:40799(127.4.197.254:40799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:13.757 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.254:40799: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:13.759 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301; will retry
07:38:13.866 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.253:37597(127.4.197.253:37597)] connecting to peer
07:38:13.868 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.254:40799(127.4.197.254:40799)] connecting to peer
07:38:13.868 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:13.869 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.197.253:37597(127.4.197.253:37597)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:13.869 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.253:37597: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:13.870 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:13.870 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.197.254:40799(127.4.197.254:40799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:13.870 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.254:40799: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:13.872 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301; will retry
07:38:14.127 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.253:37597(127.4.197.253:37597)] connecting to peer
07:38:14.128 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.254:40799(127.4.197.254:40799)] connecting to peer
07:38:14.130 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:14.130 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.197.253:37597(127.4.197.253:37597)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:14.133 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.253:37597: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:14.134 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:14.135 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.197.254:40799(127.4.197.254:40799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:14.135 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.254:40799: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:14.135 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301; will retry
07:38:14.286 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.253:37597(127.4.197.253:37597)] connecting to peer
07:38:14.287 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.254:40799(127.4.197.254:40799)] connecting to peer
07:38:14.289 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:14.289 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.197.253:37597(127.4.197.253:37597)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:14.289 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.253:37597: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:14.291 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:14.291 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.197.254:40799(127.4.197.254:40799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:14.291 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.254:40799: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:14.293 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301; will retry
07:38:14.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:14.312340 5420 raft_consensus.cc:482] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:38:14.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:14.312768 5420 raft_consensus.cc:504] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "925836e479bc498db068663e717e7b91" member_type: VOTER last_known_addr { host: "127.4.197.254" port: 40799 } } peers { permanent_uuid: "a1f04fbbf283438c901b0bae2bcb6c45" member_type: VOTER last_known_addr { host: "127.4.197.253" port: 37597 } } peers { permanent_uuid: "824f0916aaf84abeab55bd0ff85a9b64" member_type: VOTER last_known_addr { host: "127.4.197.252" port: 37301 } }
07:38:14.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:14.314625 5420 leader_election.cc:288] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 925836e479bc498db068663e717e7b91 (127.4.197.254:40799), a1f04fbbf283438c901b0bae2bcb6c45 (127.4.197.253:37597)
07:38:14.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:14.325287 5344 leader_election.cc:334] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 925836e479bc498db068663e717e7b91 (127.4.197.254:40799): Network error: Client connection negotiation failed: client connection to 127.4.197.254:40799: connect: Connection refused (error 111)
07:38:14.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:14.325999 5344 leader_election.cc:334] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a1f04fbbf283438c901b0bae2bcb6c45 (127.4.197.253:37597): Network error: Client connection negotiation failed: client connection to 127.4.197.253:37597: connect: Connection refused (error 111)
07:38:14.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:14.326270 5344 leader_election.cc:302] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 824f0916aaf84abeab55bd0ff85a9b64; no voters: 925836e479bc498db068663e717e7b91, a1f04fbbf283438c901b0bae2bcb6c45
07:38:14.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:14.326921 5420 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:38:14.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:14.596014 5411 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:38:14.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:14.596272 5411 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:38:15.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:15.599386 5411 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:38:15.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:15.599690 5411 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:38:16.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:16.476274 5428 raft_consensus.cc:482] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:38:16.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:16.476754 5428 raft_consensus.cc:504] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "925836e479bc498db068663e717e7b91" member_type: VOTER last_known_addr { host: "127.4.197.254" port: 40799 } } peers { permanent_uuid: "a1f04fbbf283438c901b0bae2bcb6c45" member_type: VOTER last_known_addr { host: "127.4.197.253" port: 37597 } } peers { permanent_uuid: "824f0916aaf84abeab55bd0ff85a9b64" member_type: VOTER last_known_addr { host: "127.4.197.252" port: 37301 } }
07:38:16.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:16.478422 5428 leader_election.cc:288] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 925836e479bc498db068663e717e7b91 (127.4.197.254:40799), a1f04fbbf283438c901b0bae2bcb6c45 (127.4.197.253:37597)
07:38:16.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:16.484656 5344 leader_election.cc:334] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 925836e479bc498db068663e717e7b91 (127.4.197.254:40799): Network error: Client connection negotiation failed: client connection to 127.4.197.254:40799: connect: Connection refused (error 111)
07:38:16.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:16.485103 5344 leader_election.cc:334] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a1f04fbbf283438c901b0bae2bcb6c45 (127.4.197.253:37597): Network error: Client connection negotiation failed: client connection to 127.4.197.253:37597: connect: Connection refused (error 111)
07:38:16.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:16.485345 5344 leader_election.cc:302] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 824f0916aaf84abeab55bd0ff85a9b64; no voters: 925836e479bc498db068663e717e7b91, a1f04fbbf283438c901b0bae2bcb6c45
07:38:16.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:16.485898 5428 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:38:16.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:16.602375 5411 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:38:16.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:16.602638 5411 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:38:17.307 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.253:37597(127.4.197.253:37597)] connecting to peer
07:38:17.308 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.254:40799(127.4.197.254:40799)] connecting to peer
07:38:17.310 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:17.310 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.197.253:37597(127.4.197.253:37597)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:17.310 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.253:37597: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:17.311 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:17.311 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.197.254:40799(127.4.197.254:40799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:17.312 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.254:40799: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:17.315 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301; will retry
07:38:17.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:17.605798 5411 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:38:17.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:17.606158 5411 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:38:17.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:17.644060 5322 master_runner.cc:186] Error getting master registration for 127.4.197.254:40799: Network error: Client connection negotiation failed: client connection to 127.4.197.254:40799: connect: Connection refused (error 111),
07:38:17.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:17.644579 5322 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:38:18.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:18.609382 5411 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:38:18.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:18.609690 5411 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:38:18.847 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.253:37597(127.4.197.253:37597)] connecting to peer
07:38:18.849 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.254:40799(127.4.197.254:40799)] connecting to peer
07:38:18.850 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:18.850 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.197.253:37597(127.4.197.253:37597)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:18.851 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.253:37597: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:18.852 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:18.852 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.197.254:40799(127.4.197.254:40799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:18.852 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.254:40799: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:18.854 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301; will retry
07:38:19.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:19.612628 5411 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:38:19.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:19.612905 5411 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:38:20.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:20.615758 5411 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:38:20.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:20.616024 5411 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:38:20.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:20.798722 5434 raft_consensus.cc:482] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:38:20.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:20.799284 5434 raft_consensus.cc:504] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "925836e479bc498db068663e717e7b91" member_type: VOTER last_known_addr { host: "127.4.197.254" port: 40799 } } peers { permanent_uuid: "a1f04fbbf283438c901b0bae2bcb6c45" member_type: VOTER last_known_addr { host: "127.4.197.253" port: 37597 } } peers { permanent_uuid: "824f0916aaf84abeab55bd0ff85a9b64" member_type: VOTER last_known_addr { host: "127.4.197.252" port: 37301 } }
07:38:20.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:20.800848 5434 leader_election.cc:288] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 925836e479bc498db068663e717e7b91 (127.4.197.254:40799), a1f04fbbf283438c901b0bae2bcb6c45 (127.4.197.253:37597)
07:38:20.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:20.803197 5344 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.197.254:40799: connect: Connection refused (error 111) [suppressed 5 similar messages]
07:38:20.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:20.806123 5344 leader_election.cc:334] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 925836e479bc498db068663e717e7b91 (127.4.197.254:40799): Network error: Client connection negotiation failed: client connection to 127.4.197.254:40799: connect: Connection refused (error 111)
07:38:20.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:20.806715 5344 leader_election.cc:334] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a1f04fbbf283438c901b0bae2bcb6c45 (127.4.197.253:37597): Network error: Client connection negotiation failed: client connection to 127.4.197.253:37597: connect: Connection refused (error 111)
07:38:20.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:20.807049 5344 leader_election.cc:302] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 824f0916aaf84abeab55bd0ff85a9b64; no voters: 925836e479bc498db068663e717e7b91, a1f04fbbf283438c901b0bae2bcb6c45
07:38:20.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:20.807688 5434 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:38:21.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:21.619462 5411 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:38:21.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:21.619851 5411 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:38:22.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:38:22 dist-test-slave-tg41 krb5kdc[4900](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790235502, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.197.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:38:22.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:22.258882 5337 init.cc:295] Successfully reacquired a new kerberos TGT
07:38:22.587 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.253:37597(127.4.197.253:37597)] connecting to peer
07:38:22.588 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.254:40799(127.4.197.254:40799)] connecting to peer
07:38:22.589 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:22.590 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.197.253:37597(127.4.197.253:37597)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:22.590 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.253:37597: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:22.591 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:22.591 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.197.254:40799(127.4.197.254:40799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:22.592 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.254:40799: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:22.594 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301; will retry
07:38:22.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:22.623018 5411 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:38:22.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:22.623366 5411 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:38:22.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:22.647671 5322 master_runner.cc:186] Error getting master registration for 127.4.197.254:40799: Network error: Client connection negotiation failed: client connection to 127.4.197.254:40799: connect: Connection refused (error 111),
07:38:22.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:22.648228 5322 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:38:23.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:23.626498 5411 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:38:23.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:23.626799 5411 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:38:24.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:24.629498 5411 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:38:24.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:24.629819 5411 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:38:25.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:25.632890 5411 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:38:25.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:25.633257 5411 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:38:26.607 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.253:37597(127.4.197.253:37597)] connecting to peer
07:38:26.608 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.254:40799(127.4.197.254:40799)] connecting to peer
07:38:26.609 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:26.609 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.197.253:37597(127.4.197.253:37597)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:26.610 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.253:37597: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:26.611 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:26.611 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.197.254:40799(127.4.197.254:40799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:26.612 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.254:40799: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:26.614 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301; will retry
07:38:26.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:26.636108 5411 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:38:26.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:26.636387 5411 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:38:27.366 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.253:37597(127.4.197.253:37597)] connecting to peer
07:38:27.367 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.254:40799(127.4.197.254:40799)] connecting to peer
07:38:27.368 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:27.369 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.197.253:37597(127.4.197.253:37597)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:27.369 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.253:37597: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:27.370 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:27.370 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.197.254:40799(127.4.197.254:40799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:27.371 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.254:40799: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:27.372 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301; will retry
07:38:27.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:27.639114 5411 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:38:27.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:27.639415 5411 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:38:27.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:27.650712 5322 master_runner.cc:186] Error getting master registration for 127.4.197.254:40799: Network error: Client connection negotiation failed: client connection to 127.4.197.254:40799: connect: Connection refused (error 111),
07:38:27.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:27.651082 5322 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:38:28.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:28.003029 5448 raft_consensus.cc:482] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:38:28.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:28.003655 5448 raft_consensus.cc:504] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "925836e479bc498db068663e717e7b91" member_type: VOTER last_known_addr { host: "127.4.197.254" port: 40799 } } peers { permanent_uuid: "a1f04fbbf283438c901b0bae2bcb6c45" member_type: VOTER last_known_addr { host: "127.4.197.253" port: 37597 } } peers { permanent_uuid: "824f0916aaf84abeab55bd0ff85a9b64" member_type: VOTER last_known_addr { host: "127.4.197.252" port: 37301 } }
07:38:28.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:28.006137 5448 leader_election.cc:288] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 925836e479bc498db068663e717e7b91 (127.4.197.254:40799), a1f04fbbf283438c901b0bae2bcb6c45 (127.4.197.253:37597)
07:38:28.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:28.009006 5344 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.197.253:37597: connect: Connection refused (error 111) [suppressed 1 similar messages]
07:38:28.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:28.012390 5344 leader_election.cc:334] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a1f04fbbf283438c901b0bae2bcb6c45 (127.4.197.253:37597): Network error: Client connection negotiation failed: client connection to 127.4.197.253:37597: connect: Connection refused (error 111)
07:38:28.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:28.013073 5344 leader_election.cc:334] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 925836e479bc498db068663e717e7b91 (127.4.197.254:40799): Network error: Client connection negotiation failed: client connection to 127.4.197.254:40799: connect: Connection refused (error 111)
07:38:28.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:28.013505 5344 leader_election.cc:302] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 824f0916aaf84abeab55bd0ff85a9b64; no voters: 925836e479bc498db068663e717e7b91, a1f04fbbf283438c901b0bae2bcb6c45
07:38:28.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:28.014329 5448 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:38:28.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:28.642861 5411 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:38:28.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:28.643210 5411 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:38:29.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:29.646260 5411 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:38:29.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:29.646625 5411 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:38:30.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:30.649475 5411 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:38:30.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:30.649814 5411 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:38:30.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:30.767633 5452 raft_consensus.cc:482] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:38:30.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:30.768151 5452 raft_consensus.cc:504] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "925836e479bc498db068663e717e7b91" member_type: VOTER last_known_addr { host: "127.4.197.254" port: 40799 } } peers { permanent_uuid: "a1f04fbbf283438c901b0bae2bcb6c45" member_type: VOTER last_known_addr { host: "127.4.197.253" port: 37597 } } peers { permanent_uuid: "824f0916aaf84abeab55bd0ff85a9b64" member_type: VOTER last_known_addr { host: "127.4.197.252" port: 37301 } }
07:38:30.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:30.769870 5452 leader_election.cc:288] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 925836e479bc498db068663e717e7b91 (127.4.197.254:40799), a1f04fbbf283438c901b0bae2bcb6c45 (127.4.197.253:37597)
07:38:30.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:30.774540 5344 leader_election.cc:334] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 925836e479bc498db068663e717e7b91 (127.4.197.254:40799): Network error: Client connection negotiation failed: client connection to 127.4.197.254:40799: connect: Connection refused (error 111)
07:38:30.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:30.775027 5344 leader_election.cc:334] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a1f04fbbf283438c901b0bae2bcb6c45 (127.4.197.253:37597): Network error: Client connection negotiation failed: client connection to 127.4.197.253:37597: connect: Connection refused (error 111)
07:38:30.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:30.775277 5344 leader_election.cc:302] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 824f0916aaf84abeab55bd0ff85a9b64; no voters: 925836e479bc498db068663e717e7b91, a1f04fbbf283438c901b0bae2bcb6c45
07:38:30.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:30.775792 5452 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:38:30.966 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.253:37597(127.4.197.253:37597)] connecting to peer
07:38:30.967 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.254:40799(127.4.197.254:40799)] connecting to peer
07:38:30.968 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:30.969 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.197.253:37597(127.4.197.253:37597)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:30.969 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.253:37597: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:30.970 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:30.970 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.197.254:40799(127.4.197.254:40799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:30.970 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.254:40799: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:30.973 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301; will retry
07:38:31.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:31.653138 5411 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:38:31.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:31.653507 5411 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:38:31.806 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.253:37597(127.4.197.253:37597)] connecting to peer
07:38:31.807 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.254:40799(127.4.197.254:40799)] connecting to peer
07:38:31.809 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:31.809 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.197.253:37597(127.4.197.253:37597)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:31.809 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.253:37597: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:31.810 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:31.810 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.197.254:40799(127.4.197.254:40799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:31.810 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.254:40799: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:31.812 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301; will retry
07:38:32.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:38:32 dist-test-slave-tg41 krb5kdc[4900](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790235512, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.197.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:38:32.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:32.261400 5337 init.cc:295] Successfully reacquired a new kerberos TGT
07:38:32.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:32.654527 5322 master_runner.cc:186] Error getting master registration for 127.4.197.254:40799: Network error: Client connection negotiation failed: client connection to 127.4.197.254:40799: connect: Connection refused (error 111),
07:38:32.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:32.654965 5322 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:38:32.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:32.656647 5411 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:38:32.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:32.656908 5411 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:38:33.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:33.660043 5411 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:38:33.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:33.660336 5411 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:38:34.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:34.663698 5411 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:38:34.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:34.664026 5411 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:38:34.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:34.865455 4239 debug-util.cc:398] Leaking SignalData structure 0x7b08000a86e0 after lost signal to thread 4176
07:38:34.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:34.866078 4239 debug-util.cc:398] Leaking SignalData structure 0x7b080009a5a0 after lost signal to thread 4242
07:38:35.627 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.253:37597(127.4.197.253:37597)] connecting to peer
07:38:35.628 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.254:40799(127.4.197.254:40799)] connecting to peer
07:38:35.628 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:35.629 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.197.253:37597(127.4.197.253:37597)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:35.629 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.253:37597: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:35.630 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:35.630 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.197.254:40799(127.4.197.254:40799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:35.631 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.254:40799: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:35.635 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301; will retry
07:38:35.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:35.668542 5411 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:38:35.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:35.668895 5411 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:38:36.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:36.671823 5411 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:38:36.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:36.672151 5411 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:38:37.146 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.253:37597(127.4.197.253:37597)] connecting to peer
07:38:37.147 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.254:40799(127.4.197.254:40799)] connecting to peer
07:38:37.147 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:37.148 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.197.253:37597(127.4.197.253:37597)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:37.148 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.253:37597: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:37.148 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:37.148 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.197.254:40799(127.4.197.254:40799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:37.149 [INFO - pool-6-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.254:40799: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:37.152 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301; will retry
07:38:37.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:37.657644 5322 master_runner.cc:186] Error getting master registration for 127.4.197.254:40799: Network error: Client connection negotiation failed: client connection to 127.4.197.254:40799: connect: Connection refused (error 111),
07:38:37.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:37.658037 5322 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:38:37.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:37.675052 5411 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:38:37.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:37.675357 5411 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:38:38.327 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.253:37597(127.4.197.253:37597)] connecting to peer
07:38:38.328 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.254:40799(127.4.197.254:40799)] connecting to peer
07:38:38.329 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:38.329 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.197.253:37597(127.4.197.253:37597)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:38.329 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.253:37597: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:38.329 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:38.330 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.197.254:40799(127.4.197.254:40799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:38.330 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.254:40799: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:38.332 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301; will retry
07:38:38.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:38.678175 5411 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:38:38.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:38.678486 5411 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:38:38.726 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.253:37597(127.4.197.253:37597)] connecting to peer
07:38:38.727 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.254:40799(127.4.197.254:40799)] connecting to peer
07:38:38.727 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:38.727 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.197.253:37597(127.4.197.253:37597)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:38.728 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.253:37597: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:38.728 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:38.729 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.197.254:40799(127.4.197.254:40799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:38.729 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.254:40799: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:38.731 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301; will retry
07:38:38.766 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.253:37597(127.4.197.253:37597)] connecting to peer
07:38:38.767 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.254:40799(127.4.197.254:40799)] connecting to peer
07:38:38.768 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:38.768 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.197.253:37597(127.4.197.253:37597)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:38.768 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.253:37597: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:38.769 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:38.769 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.197.254:40799(127.4.197.254:40799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:38.769 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.254:40799: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:38.771 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301; will retry
07:38:39.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:39.681203 5411 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:38:39.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:39.681514 5411 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:38:40.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:40.684389 5411 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:38:40.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:40.684638 5411 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:38:41.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:41.687932 5411 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 824f0916aaf84abeab55bd0ff85a9b64: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:38:41.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:38:41.688252 5411 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:38:42.186 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.253:37597(127.4.197.253:37597)] connecting to peer
07:38:42.187 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.197.254:40799(127.4.197.254:40799)] connecting to peer
07:38:42.188 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:42.188 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.197.253:37597(127.4.197.253:37597)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:42.188 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.253:37597: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597
07:38:42.189 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:42.189 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.197.254:40799(127.4.197.254:40799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:42.189 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.197.254:40799: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
07:38:42.193 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301; will retry
07:38:42.199 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=29748), Traces: [0ms] refreshing cache from master, [33ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [107ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [109ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [194ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [196ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [572ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [577ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [598ms] refreshing cache from master, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [601ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [601ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [606ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [608ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [611ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [612ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [617ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [620ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [624ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [626ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [627ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [639ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [645ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [648ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [649ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [677ms] refreshing cache from master, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [680ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [681ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [685ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [686ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [687ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [717ms] refreshing cache from master, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [721ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [722ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [735ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [737ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [740ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [817ms] refreshing cache from master, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [820ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [821ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [826ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [827ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [830ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [877ms] refreshing cache from master, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [880ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [881ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [884ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [886ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [888ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [1017ms] refreshing cache from master, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [1018ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [1019ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [1020ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [1023ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [1024ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [1028ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [1297ms] refreshing cache from master, [1297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [1298ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [1299ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [1300ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [1306ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [1309ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [1310ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [1416ms] refreshing cache from master, [1417ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [1418ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [1419ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [1420ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [1421ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [1423ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [1423ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [1677ms] refreshing cache from master, [1677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [1679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [1679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [1681ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [1685ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [1686ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [1686ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [1837ms] refreshing cache from master, [1837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [1838ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [1839ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [1840ms] too many traces: truncated at 100 traces, deferred=Deferred@1157863593(state=PENDING, result=null, callback=(continuation of Deferred@526590049 after retry RPC after error@1639445259) -> (continuation of Deferred@1405902449 after retry RPC after error@173232753) -> (continuation of Deferred@213191538 after retry RPC after error@1380224580) -> (continuation of Deferred@1674214827 after retry RPC after error@1931111289) -> (continuation of Deferred@1464865011 after retry RPC after error@563122931) -> (continuation of Deferred@294097999 after retry RPC after error@828798117) -> (continuation of Deferred@1726365170 after retry RPC after error@1128541399) -> (continuation of Deferred@1850535196 after retry RPC after error@1812823852) -> (continuation of Deferred@1003734770 after retry RPC after error@1502836683) -> (continuation of Deferred@1354651112 after retry RPC after error@1473098956) -> (continuation of Deferred@1921784977 after retry RPC after error@811798564) -> (continuation of Deferred@315622101 after retry RPC after error@671203223) -> (continuation of Deferred@981161618 after retry RPC after error@305238985) -> (continuation of Deferred@6684144 after retry RPC after error@624760684) -> (continuation of Deferred@1167202975 after retry RPC after error@1537435199) -> (continuation of Deferred@1954974406 after retry RPC after error@969766221) -> (continuation of Deferred@1818150409 after retry RPC after error@739034915) -> (continuation of Deferred@1822210270 after retry RPC after error@1042931485) -> (continuation of Deferred@618477663 after retry RPC after error@1847772728) -> (continuation of Deferred@1735514474 after retry RPC after error@399520686) -> (continuation of Deferred@1518530251 after retry RPC after error@585756837) -> (continuation of Deferred@1270531867 after retry RPC after error@1566691457) -> (continuation of Deferred@282630480 after retry RPC after error@2111385398) -> (continuation of Deferred@1615248777 after retry RPC after error@871402712) -> (continuation of Deferred@1731625455 after retry RPC after error@1394778784), errback=(continuation of Deferred@526590049 after retry RPC after error@1639445259) -> (continuation of Deferred@1405902449 after retry RPC after error@173232753) -> (continuation of Deferred@213191538 after retry RPC after error@1380224580) -> (continuation of Deferred@1674214827 after retry RPC after error@1931111289) -> (continuation of Deferred@1464865011 after retry RPC after error@563122931) -> (continuation of Deferred@294097999 after retry RPC after error@828798117) -> (continuation of Deferred@1726365170 after retry RPC after error@1128541399) -> (continuation of Deferred@1850535196 after retry RPC after error@1812823852) -> (continuation of Deferred@1003734770 after retry RPC after error@1502836683) -> (continuation of Deferred@1354651112 after retry RPC after error@1473098956) -> (continuation of Deferred@1921784977 after retry RPC after error@811798564) -> (continuation of Deferred@315622101 after retry RPC after error@671203223) -> (continuation of Deferred@981161618 after retry RPC after error@305238985) -> (continuation of Deferred@6684144 after retry RPC after error@624760684) -> (continuation of Deferred@1167202975 after retry RPC after error@1537435199) -> (continuation of Deferred@1954974406 after retry RPC after error@969766221) -> (continuation of Deferred@1818150409 after retry RPC after error@739034915) -> (continuation of Deferred@1822210270 after retry RPC after error@1042931485) -> (continuation of Deferred@618477663 after retry RPC after error@1847772728) -> (continuation of Deferred@1735514474 after retry RPC after error@399520686) -> (continuation of Deferred@1518530251 after retry RPC after error@585756837) -> (continuation of Deferred@1270531867 after retry RPC after error@1566691457) -> (continuation of Deferred@282630480 after retry RPC after error@2111385398) -> (continuation of Deferred@1615248777 after retry RPC after error@871402712) -> (continuation of Deferred@1731625455 after retry RPC after error@1394778784)))
07:38:42.204 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.4.197.252:37301(127.4.197.252:37301)] disconnecting while in state READY
07:38:42.205 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.4.197.254:40799(127.4.197.254:40799)] disconnecting while in state TERMINATED
07:38:42.206 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.4.197.253:37597(127.4.197.253:37597)] disconnecting while in state TERMINATED
07:38:42.212 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
07:38:42.214 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.4.197.252:37301(127.4.197.252:37301)] handling channelInactive
07:38:42.214 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
07:38:42.214 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.197.252:37301(127.4.197.252:37301)] cleaning up while in state READY due to: connection closed
07:38:42.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:42.218430 4887 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskDxb_lc/build/tsan/bin/kudu with pid 5322
07:38:43.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-24T07:38:43Z chronyd exiting
07:38:44.463 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
07:38:44.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:44.464835 4163 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskDxb_lc/build/tsan/bin/kudu with pid 4443
07:38:44.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:44.490294 4163 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskDxb_lc/build/tsan/bin/kudu with pid 4603
07:38:44.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:44.516260 4163 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskDxb_lc/build/tsan/bin/kudu with pid 4746
07:38:44.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:44.544546 4163 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskDxb_lc/build/tsan/bin/kudu with pid 4175
07:38:44.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:44.577322 4163 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskDxb_lc/build/tsan/bin/kudu with pid 4245
07:38:44.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:38:44.608799 4163 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskDxb_lc/build/tsan/bin/kudu with pid 4320
07:38:45.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-24T07:38:45Z chronyd exiting
07:38:46.804 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=29748), Traces: [0ms] refreshing cache from master, [33ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [107ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [109ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [194ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [196ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [572ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [577ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [598ms] refreshing cache from master, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [601ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [601ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [606ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [608ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [611ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [612ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [617ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [620ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [624ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [626ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [627ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [639ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [645ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [648ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [649ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [677ms] refreshing cache from master, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [680ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [681ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [685ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [686ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [687ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [717ms] refreshing cache from master, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [721ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [722ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [735ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [737ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [740ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [817ms] refreshing cache from master, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [820ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [821ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [826ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [827ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [830ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [877ms] refreshing cache from master, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [880ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [881ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [884ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [886ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [888ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [1017ms] refreshing cache from master, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [1018ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [1019ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [1020ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [1023ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [1024ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [1028ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [1297ms] refreshing cache from master, [1297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [1298ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [1299ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [1300ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [1306ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [1309ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [1310ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [1416ms] refreshing cache from master, [1417ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [1418ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [1419ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [1420ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [1421ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [1423ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [1423ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [1677ms] refreshing cache from master, [1677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [1679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [1679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [1681ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [1685ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [1686ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [1686ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [1837ms] refreshing cache from master, [1837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [1838ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [1839ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [1840ms] too many traces: truncated at 100 traces, deferred=Deferred@1157863593(state=PENDING, result=null, callback=(continuation of Deferred@526590049 after retry RPC after error@1639445259) -> (continuation of Deferred@1405902449 after retry RPC after error@173232753) -> (continuation of Deferred@213191538 after retry RPC after error@1380224580) -> (continuation of Deferred@1674214827 after retry RPC after error@1931111289) -> (continuation of Deferred@1464865011 after retry RPC after error@563122931) -> (continuation of Deferred@294097999 after retry RPC after error@828798117) -> (continuation of Deferred@1726365170 after retry RPC after error@1128541399) -> (continuation of Deferred@1850535196 after retry RPC after error@1812823852) -> (continuation of Deferred@1003734770 after retry RPC after error@1502836683) -> (continuation of Deferred@1354651112 after retry RPC after error@1473098956) -> (continuation of Deferred@1921784977 after retry RPC after error@811798564) -> (continuation of Deferred@315622101 after retry RPC after error@671203223) -> (continuation of Deferred@981161618 after retry RPC after error@305238985) -> (continuation of Deferred@6684144 after retry RPC after error@624760684) -> (continuation of Deferred@1167202975 after retry RPC after error@1537435199) -> (continuation of Deferred@1954974406 after retry RPC after error@969766221) -> (continuation of Deferred@1818150409 after retry RPC after error@739034915) -> (continuation of Deferred@1822210270 after retry RPC after error@1042931485) -> (continuation of Deferred@618477663 after retry RPC after error@1847772728) -> (continuation of Deferred@1735514474 after retry RPC after error@399520686) -> (continuation of Deferred@1518530251 after retry RPC after error@585756837) -> (continuation of Deferred@1270531867 after retry RPC after error@1566691457) -> (continuation of Deferred@282630480 after retry RPC after error@2111385398) -> (continuation of Deferred@1615248777 after retry RPC after error@871402712) -> (continuation of Deferred@1731625455 after retry RPC after error@1394778784), errback=(continuation of Deferred@526590049 after retry RPC after error@1639445259) -> (continuation of Deferred@1405902449 after retry RPC after error@173232753) -> (continuation of Deferred@213191538 after retry RPC after error@1380224580) -> (continuation of Deferred@1674214827 after retry RPC after error@1931111289) -> (continuation of Deferred@1464865011 after retry RPC after error@563122931) -> (continuation of Deferred@294097999 after retry RPC after error@828798117) -> (continuation of Deferred@1726365170 after retry RPC after error@1128541399) -> (continuation of Deferred@1850535196 after retry RPC after error@1812823852) -> (continuation of Deferred@1003734770 after retry RPC after error@1502836683) -> (continuation of Deferred@1354651112 after retry RPC after error@1473098956) -> (continuation of Deferred@1921784977 after retry RPC after error@811798564) -> (continuation of Deferred@315622101 after retry RPC after error@671203223) -> (continuation of Deferred@981161618 after retry RPC after error@305238985) -> (continuation of Deferred@6684144 after retry RPC after error@624760684) -> (continuation of Deferred@1167202975 after retry RPC after error@1537435199) -> (continuation of Deferred@1954974406 after retry RPC after error@969766221) -> (continuation of Deferred@1818150409 after retry RPC after error@739034915) -> (continuation of Deferred@1822210270 after retry RPC after error@1042931485) -> (continuation of Deferred@618477663 after retry RPC after error@1847772728) -> (continuation of Deferred@1735514474 after retry RPC after error@399520686) -> (continuation of Deferred@1518530251 after retry RPC after error@585756837) -> (continuation of Deferred@1270531867 after retry RPC after error@1566691457) -> (continuation of Deferred@282630480 after retry RPC after error@2111385398) -> (continuation of Deferred@1615248777 after retry RPC after error@871402712) -> (continuation of Deferred@1731625455 after retry RPC after error@1394778784)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.4.197.254:40799
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=29748), Traces: [0ms] refreshing cache from master, [33ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [107ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [109ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [194ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [196ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [572ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [577ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [598ms] refreshing cache from master, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [601ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [601ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [606ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [608ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [611ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [612ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [617ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [620ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [624ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [626ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [627ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [639ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [645ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [648ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [649ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [677ms] refreshing cache from master, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [680ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [681ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [685ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [686ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [687ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [717ms] refreshing cache from master, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [721ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [722ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [735ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [737ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [740ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [817ms] refreshing cache from master, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [820ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [821ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [826ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [827ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [830ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [877ms] refreshing cache from master, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [880ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [881ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [884ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [886ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [888ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [1017ms] refreshing cache from master, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [1018ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [1019ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [1020ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [1023ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [1024ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [1028ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [1297ms] refreshing cache from master, [1297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [1298ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [1299ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [1300ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [1306ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [1309ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [1310ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [1416ms] refreshing cache from master, [1417ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [1418ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [1419ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [1420ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [1421ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [1423ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [1423ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [1677ms] refreshing cache from master, [1677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [1679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [1679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [1681ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [1685ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [1686ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [1686ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [1837ms] refreshing cache from master, [1837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [1838ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [1839ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [1840ms] too many traces: truncated at 100 traces, deferred=Deferred@1157863593(state=PENDING, result=null, callback=(continuation of Deferred@526590049 after retry RPC after error@1639445259) -> (continuation of Deferred@1405902449 after retry RPC after error@173232753) -> (continuation of Deferred@213191538 after retry RPC after error@1380224580) -> (continuation of Deferred@1674214827 after retry RPC after error@1931111289) -> (continuation of Deferred@1464865011 after retry RPC after error@563122931) -> (continuation of Deferred@294097999 after retry RPC after error@828798117) -> (continuation of Deferred@1726365170 after retry RPC after error@1128541399) -> (continuation of Deferred@1850535196 after retry RPC after error@1812823852) -> (continuation of Deferred@1003734770 after retry RPC after error@1502836683) -> (continuation of Deferred@1354651112 after retry RPC after error@1473098956) -> (continuation of Deferred@1921784977 after retry RPC after error@811798564) -> (continuation of Deferred@315622101 after retry RPC after error@671203223) -> (continuation of Deferred@981161618 after retry RPC after error@305238985) -> (continuation of Deferred@6684144 after retry RPC after error@624760684) -> (continuation of Deferred@1167202975 after retry RPC after error@1537435199) -> (continuation of Deferred@1954974406 after retry RPC after error@969766221) -> (continuation of Deferred@1818150409 after retry RPC after error@739034915) -> (continuation of Deferred@1822210270 after retry RPC after error@1042931485) -> (continuation of Deferred@618477663 after retry RPC after error@1847772728) -> (continuation of Deferred@1735514474 after retry RPC after error@399520686) -> (continuation of Deferred@1518530251 after retry RPC after error@585756837) -> (continuation of Deferred@1270531867 after retry RPC after error@1566691457) -> (continuation of Deferred@282630480 after retry RPC after error@2111385398) -> (continuation of Deferred@1615248777 after retry RPC after error@871402712) -> (continuation of Deferred@1731625455 after retry RPC after error@1394778784), errback=(continuation of Deferred@526590049 after retry RPC after error@1639445259) -> (continuation of Deferred@1405902449 after retry RPC after error@173232753) -> (continuation of Deferred@213191538 after retry RPC after error@1380224580) -> (continuation of Deferred@1674214827 after retry RPC after error@1931111289) -> (continuation of Deferred@1464865011 after retry RPC after error@563122931) -> (continuation of Deferred@294097999 after retry RPC after error@828798117) -> (continuation of Deferred@1726365170 after retry RPC after error@1128541399) -> (continuation of Deferred@1850535196 after retry RPC after error@1812823852) -> (continuation of Deferred@1003734770 after retry RPC after error@1502836683) -> (continuation of Deferred@1354651112 after retry RPC after error@1473098956) -> (continuation of Deferred@1921784977 after retry RPC after error@811798564) -> (continuation of Deferred@315622101 after retry RPC after error@671203223) -> (continuation of Deferred@981161618 after retry RPC after error@305238985) -> (continuation of Deferred@6684144 after retry RPC after error@624760684) -> (continuation of Deferred@1167202975 after retry RPC after error@1537435199) -> (continuation of Deferred@1954974406 after retry RPC after error@969766221) -> (continuation of Deferred@1818150409 after retry RPC after error@739034915) -> (continuation of Deferred@1822210270 after retry RPC after error@1042931485) -> (continuation of Deferred@618477663 after retry RPC after error@1847772728) -> (continuation of Deferred@1735514474 after retry RPC after error@399520686) -> (continuation of Deferred@1518530251 after retry RPC after error@585756837) -> (continuation of Deferred@1270531867 after retry RPC after error@1566691457) -> (continuation of Deferred@282630480 after retry RPC after error@2111385398) -> (continuation of Deferred@1615248777 after retry RPC after error@871402712) -> (continuation of Deferred@1731625455 after retry RPC after error@1394778784)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.4.197.254:40799
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
07:38:48.024 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
07:38:48.024 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=29748), Traces: [0ms] refreshing cache from master, [33ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [107ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [109ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [194ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [196ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [572ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [577ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [598ms] refreshing cache from master, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [601ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [601ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [606ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [608ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [611ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [612ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [617ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [620ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [624ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [626ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [627ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [639ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [645ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [648ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [649ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [677ms] refreshing cache from master, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [680ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [681ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [685ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [686ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [687ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [717ms] refreshing cache from master, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [721ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [722ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [735ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [737ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [740ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [817ms] refreshing cache from master, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [820ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [821ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [826ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [827ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [830ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [877ms] refreshing cache from master, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [880ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [881ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [884ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [886ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [888ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [1017ms] refreshing cache from master, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [1018ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [1019ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [1020ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [1023ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [1024ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [1028ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [1297ms] refreshing cache from master, [1297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [1298ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [1299ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [1300ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [1306ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [1309ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [1310ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [1416ms] refreshing cache from master, [1417ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [1418ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [1419ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [1420ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [1421ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [1423ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [1423ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [1677ms] refreshing cache from master, [1677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [1679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [1679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [1681ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [1685ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [1686ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [1686ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [1837ms] refreshing cache from master, [1837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [1838ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [1839ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [1840ms] too many traces: truncated at 100 traces, deferred=Deferred@1157863593(state=PENDING, result=null, callback=(continuation of Deferred@526590049 after retry RPC after error@1639445259) -> (continuation of Deferred@1405902449 after retry RPC after error@173232753) -> (continuation of Deferred@213191538 after retry RPC after error@1380224580) -> (continuation of Deferred@1674214827 after retry RPC after error@1931111289) -> (continuation of Deferred@1464865011 after retry RPC after error@563122931) -> (continuation of Deferred@294097999 after retry RPC after error@828798117) -> (continuation of Deferred@1726365170 after retry RPC after error@1128541399) -> (continuation of Deferred@1850535196 after retry RPC after error@1812823852) -> (continuation of Deferred@1003734770 after retry RPC after error@1502836683) -> (continuation of Deferred@1354651112 after retry RPC after error@1473098956) -> (continuation of Deferred@1921784977 after retry RPC after error@811798564) -> (continuation of Deferred@315622101 after retry RPC after error@671203223) -> (continuation of Deferred@981161618 after retry RPC after error@305238985) -> (continuation of Deferred@6684144 after retry RPC after error@624760684) -> (continuation of Deferred@1167202975 after retry RPC after error@1537435199) -> (continuation of Deferred@1954974406 after retry RPC after error@969766221) -> (continuation of Deferred@1818150409 after retry RPC after error@739034915) -> (continuation of Deferred@1822210270 after retry RPC after error@1042931485) -> (continuation of Deferred@618477663 after retry RPC after error@1847772728) -> (continuation of Deferred@1735514474 after retry RPC after error@399520686) -> (continuation of Deferred@1518530251 after retry RPC after error@585756837) -> (continuation of Deferred@1270531867 after retry RPC after error@1566691457) -> (continuation of Deferred@282630480 after retry RPC after error@2111385398) -> (continuation of Deferred@1615248777 after retry RPC after error@871402712) -> (continuation of Deferred@1731625455 after retry RPC after error@1394778784), errback=(continuation of Deferred@526590049 after retry RPC after error@1639445259) -> (continuation of Deferred@1405902449 after retry RPC after error@173232753) -> (continuation of Deferred@213191538 after retry RPC after error@1380224580) -> (continuation of Deferred@1674214827 after retry RPC after error@1931111289) -> (continuation of Deferred@1464865011 after retry RPC after error@563122931) -> (continuation of Deferred@294097999 after retry RPC after error@828798117) -> (continuation of Deferred@1726365170 after retry RPC after error@1128541399) -> (continuation of Deferred@1850535196 after retry RPC after error@1812823852) -> (continuation of Deferred@1003734770 after retry RPC after error@1502836683) -> (continuation of Deferred@1354651112 after retry RPC after error@1473098956) -> (continuation of Deferred@1921784977 after retry RPC after error@811798564) -> (continuation of Deferred@315622101 after retry RPC after error@671203223) -> (continuation of Deferred@981161618 after retry RPC after error@305238985) -> (continuation of Deferred@6684144 after retry RPC after error@624760684) -> (continuation of Deferred@1167202975 after retry RPC after error@1537435199) -> (continuation of Deferred@1954974406 after retry RPC after error@969766221) -> (continuation of Deferred@1818150409 after retry RPC after error@739034915) -> (continuation of Deferred@1822210270 after retry RPC after error@1042931485) -> (continuation of Deferred@618477663 after retry RPC after error@1847772728) -> (continuation of Deferred@1735514474 after retry RPC after error@399520686) -> (continuation of Deferred@1518530251 after retry RPC after error@585756837) -> (continuation of Deferred@1270531867 after retry RPC after error@1566691457) -> (continuation of Deferred@282630480 after retry RPC after error@2111385398) -> (continuation of Deferred@1615248777 after retry RPC after error@871402712) -> (continuation of Deferred@1731625455 after retry RPC after error@1394778784)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.4.197.254:40799
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=29748), Traces: [0ms] refreshing cache from master, [33ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [107ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [109ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [194ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [196ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [572ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [577ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [598ms] refreshing cache from master, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [601ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [601ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [606ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [608ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [611ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [612ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [617ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [620ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [624ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [626ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [627ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [639ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [645ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [648ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [649ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [677ms] refreshing cache from master, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [680ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [681ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [685ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [686ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [687ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [717ms] refreshing cache from master, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [721ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [722ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [735ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [737ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [740ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [817ms] refreshing cache from master, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [820ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [821ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [826ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [827ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [830ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [877ms] refreshing cache from master, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [880ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [881ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [884ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [886ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [888ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [1017ms] refreshing cache from master, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [1018ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [1019ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [1020ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [1023ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [1024ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [1028ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [1297ms] refreshing cache from master, [1297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [1298ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [1299ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [1300ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [1306ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [1309ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [1310ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [1416ms] refreshing cache from master, [1417ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [1418ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [1419ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [1420ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [1421ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [1423ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [1423ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [1677ms] refreshing cache from master, [1677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [1679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [1679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [1681ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.253:37597: Network error: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597, [1685ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.252:37301: OK, [1686ms] Sub RPC ConnectToMaster: received response from server master-127.4.197.254:40799: Network error: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [1686ms] delaying RPC due to: Service unavailable: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799, [1837ms] refreshing cache from master, [1837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.253:37597, [1838ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.254:40799, [1839ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.197.252:37301, [1840ms] too many traces: truncated at 100 traces, deferred=Deferred@1157863593(state=PENDING, result=null, callback=(continuation of Deferred@526590049 after retry RPC after error@1639445259) -> (continuation of Deferred@1405902449 after retry RPC after error@173232753) -> (continuation of Deferred@213191538 after retry RPC after error@1380224580) -> (continuation of Deferred@1674214827 after retry RPC after error@1931111289) -> (continuation of Deferred@1464865011 after retry RPC after error@563122931) -> (continuation of Deferred@294097999 after retry RPC after error@828798117) -> (continuation of Deferred@1726365170 after retry RPC after error@1128541399) -> (continuation of Deferred@1850535196 after retry RPC after error@1812823852) -> (continuation of Deferred@1003734770 after retry RPC after error@1502836683) -> (continuation of Deferred@1354651112 after retry RPC after error@1473098956) -> (continuation of Deferred@1921784977 after retry RPC after error@811798564) -> (continuation of Deferred@315622101 after retry RPC after error@671203223) -> (continuation of Deferred@981161618 after retry RPC after error@305238985) -> (continuation of Deferred@6684144 after retry RPC after error@624760684) -> (continuation of Deferred@1167202975 after retry RPC after error@1537435199) -> (continuation of Deferred@1954974406 after retry RPC after error@969766221) -> (continuation of Deferred@1818150409 after retry RPC after error@739034915) -> (continuation of Deferred@1822210270 after retry RPC after error@1042931485) -> (continuation of Deferred@618477663 after retry RPC after error@1847772728) -> (continuation of Deferred@1735514474 after retry RPC after error@399520686) -> (continuation of Deferred@1518530251 after retry RPC after error@585756837) -> (continuation of Deferred@1270531867 after retry RPC after error@1566691457) -> (continuation of Deferred@282630480 after retry RPC after error@2111385398) -> (continuation of Deferred@1615248777 after retry RPC after error@871402712) -> (continuation of Deferred@1731625455 after retry RPC after error@1394778784), errback=(continuation of Deferred@526590049 after retry RPC after error@1639445259) -> (continuation of Deferred@1405902449 after retry RPC after error@173232753) -> (continuation of Deferred@213191538 after retry RPC after error@1380224580) -> (continuation of Deferred@1674214827 after retry RPC after error@1931111289) -> (continuation of Deferred@1464865011 after retry RPC after error@563122931) -> (continuation of Deferred@294097999 after retry RPC after error@828798117) -> (continuation of Deferred@1726365170 after retry RPC after error@1128541399) -> (continuation of Deferred@1850535196 after retry RPC after error@1812823852) -> (continuation of Deferred@1003734770 after retry RPC after error@1502836683) -> (continuation of Deferred@1354651112 after retry RPC after error@1473098956) -> (continuation of Deferred@1921784977 after retry RPC after error@811798564) -> (continuation of Deferred@315622101 after retry RPC after error@671203223) -> (continuation of Deferred@981161618 after retry RPC after error@305238985) -> (continuation of Deferred@6684144 after retry RPC after error@624760684) -> (continuation of Deferred@1167202975 after retry RPC after error@1537435199) -> (continuation of Deferred@1954974406 after retry RPC after error@969766221) -> (continuation of Deferred@1818150409 after retry RPC after error@739034915) -> (continuation of Deferred@1822210270 after retry RPC after error@1042931485) -> (continuation of Deferred@618477663 after retry RPC after error@1847772728) -> (continuation of Deferred@1735514474 after retry RPC after error@399520686) -> (continuation of Deferred@1518530251 after retry RPC after error@585756837) -> (continuation of Deferred@1270531867 after retry RPC after error@1566691457) -> (continuation of Deferred@282630480 after retry RPC after error@2111385398) -> (continuation of Deferred@1615248777 after retry RPC after error@871402712) -> (continuation of Deferred@1731625455 after retry RPC after error@1394778784)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.4.197.253:37597,127.4.197.254:40799,127.4.197.252:37301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.253:37597(127.4.197.253:37597): Connection refused: /127.4.197.253:37597,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.197.254:40799(127.4.197.254:40799): Connection refused: /127.4.197.254:40799
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.4.197.254:40799
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)