05:26:19.283 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
05:26:19.295 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task_eGFIx/test-tmp
05:26:19.298 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
05:26:19.301 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task_eGFIx/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
05:26:19.505 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
05:26:19.700 [DEBUG - main] (MiniKuduCluster.java:195) Response:
05:26:19.700 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
05:26:19.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:19.704073 374 mini_oidc.cc:152] Starting JWKS server
05:26:19.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:19.707988 374 webserver.cc:460] Webserver started at http://127.0.0.1:34201/ using document root <none> and password file <none>
05:26:19.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:19.708526 374 mini_oidc.cc:189] Starting OIDC Discovery server
05:26:19.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:19.710218 374 webserver.cc:460] Webserver started at http://127.0.0.1:38243/ using document root <none> and password file <none>
05:26:19.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T05:26:19Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
05:26:19.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T05:26:19Z Disabled control of system clock
05:26:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:19.759694 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-task_eGFIx/build/tsan/bin/kudu
05:26:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_eGFIx/build/tsan/bin/kudu
05:26:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-0/wal
05:26:19.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-0/data
05:26:19.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:26:19.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:26:19.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:26:19.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:26:19.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:26:19.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:26:19.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-0/logs
05:26:19.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-0/data/info.pb
05:26:19.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:26:19.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:26:19.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:26:19.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:26:19.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:26:19.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:26:19.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:26:19.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:26:19.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:26:19.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:26:19.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:26:19.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:26:19.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.190:42033
05:26:19.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.190
05:26:19.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:26:19.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:46351
05:26:19.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:26:19.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:26:19.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:26:19.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:26:19.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:42033,127.0.93.189:35555,127.0.93.188:41061
05:26:19.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:26:19.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34201/jwks.json with env {}
05:26:20.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:20.062317 386 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:26:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:20.062872 386 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:26:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:20.063512 386 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:26:20.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:20.093627 386 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:26:20.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:20.093894 386 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:26:20.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:20.094180 386 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:26:20.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:20.094485 386 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:26:20.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:20.094786 386 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:26:20.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:20.095018 386 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34201/jwks.json
05:26:20.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:20.129850 386 master_runner.cc:386] Master server non-default flags:
05:26:20.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:26:20.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:46351
05:26:20.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:26:20.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:26:20.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-0/data
05:26:20.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-0/wal
05:26:20.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:26:20.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:42033,127.0.93.189:35555,127.0.93.188:41061
05:26:20.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:26:20.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:26:20.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:26:20.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.190:42033
05:26:20.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:26:20.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:26:20.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:26:20.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34201/jwks.json
05:26:20.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:26:20.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:26:20.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-0/data/info.pb
05:26:20.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.190
05:26:20.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:26:20.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:26:20.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:26:20.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:26:20.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:26:20.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:26:20.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-0/logs
05:26:20.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:26:20.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:26:20.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:26:20.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:26:20.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:26:20.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:26:20.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:26:20.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 04:01:01 UTC on ccycloud.quasar-phqekz.root.comops.site
05:26:20.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679192
05:26:20.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:26:20.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:20.131175 386 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:26:20.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:20.132893 386 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:26:20.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:20.152168 392 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:26:21.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:20.152200 395 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:26:21.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:20.154188 386 server_base.cc:1034] running on GCE node
05:26:21.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:20.152207 393 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:26:21.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:21.329632 386 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:26:21.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:21.333652 386 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:26:21.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:21.335157 386 hybrid_clock.cc:648] HybridClock initialized: now 1790659581335101 us; error 76 us; skew 500 ppm
05:26:21.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:21.336169 386 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:26:21.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:21.343353 386 webserver.cc:460] Webserver started at http://127.0.93.190:38567/ using document root <none> and password file <none>
05:26:21.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:21.344384 386 fs_manager.cc:359] Metadata directory not provided
05:26:21.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:21.344581 386 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:26:21.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:21.344992 386 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:26:21.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:21.350713 386 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-0/data/instance:
05:26:21.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e5f93bbf8c6e414490f1b5e236d7830f"
05:26:21.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:26:21 on dist-test-slave-gvjk"
05:26:21.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:21.352010 386 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-0/wal/instance:
05:26:21.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e5f93bbf8c6e414490f1b5e236d7830f"
05:26:21.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:26:21 on dist-test-slave-gvjk"
05:26:21.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:21.359747 386 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
05:26:21.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:21.365823 402 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:26:21.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:21.366923 386 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
05:26:21.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:21.367197 386 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-0/data,/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-0/wal
05:26:21.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e5f93bbf8c6e414490f1b5e236d7830f"
05:26:21.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:26:21 on dist-test-slave-gvjk"
05:26:21.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:21.367477 386 fs_report.cc:383] FS layout report
05:26:21.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:26:21.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-0/wal
05:26:21.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-0/wal
05:26:21.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-0/data/data
05:26:21.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:26:21.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:26:21.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:26:21.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:26:21.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:26:21.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:26:21.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:26:21.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:26:21.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:26:21.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:26:21.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:26:21.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:21.403218 386 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:26:21.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:21.404628 386 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:26:21.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:21.405033 386 kserver.cc:163] Server-wide thread pool size limit: 3276
05:26:21.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:21.477099 386 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.190:42033
05:26:21.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:21.477200 453 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.190:42033 every 8 connection(s)
05:26:21.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:21.479630 386 server_base.cc:1166] Dumped server information to /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-0/data/info.pb
05:26:21.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:21.485062 454 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:26:21.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:21.489370 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-task_eGFIx/build/tsan/bin/kudu as pid 386
05:26:21.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:21.489830 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-0/wal/instance
05:26:21.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:21.493348 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-task_eGFIx/build/tsan/bin/kudu
05:26:21.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_eGFIx/build/tsan/bin/kudu
05:26:21.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-1/wal
05:26:21.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-1/data
05:26:21.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:26:21.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:26:21.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:26:21.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:26:21.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:26:21.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:26:21.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-1/logs
05:26:21.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-1/data/info.pb
05:26:21.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:26:21.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:26:21.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:26:21.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:26:21.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:26:21.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:26:21.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:26:21.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:26:21.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:26:21.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:26:21.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:26:21.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:26:21.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.189:35555
05:26:21.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.189
05:26:21.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:26:21.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:46351
05:26:21.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:26:21.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:26:21.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:26:21.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:26:21.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:42033,127.0.93.189:35555,127.0.93.188:41061
05:26:21.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:26:21.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34201/jwks.json with env {}
05:26:21.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:21.498399 454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42033 } has no permanent_uuid. Determining permanent_uuid...
05:26:21.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:21.527724 454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.189" port: 35555 } has no permanent_uuid. Determining permanent_uuid...
05:26:21.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:21.529911 406 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.93.189:35555: connect: Connection refused (error 111)
05:26:21.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:21.534070 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:35555: Network error: Client connection negotiation failed: client connection to 127.0.93.189:35555: connect: Connection refused (error 111)
05:26:21.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:21.583490 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 35555 } attempt: 1
05:26:21.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:21.588042 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:35555: Network error: Client connection negotiation failed: client connection to 127.0.93.189:35555: connect: Connection refused (error 111)
05:26:21.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:21.656450 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 35555 } attempt: 2
05:26:21.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:21.660315 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:35555: Network error: Client connection negotiation failed: client connection to 127.0.93.189:35555: connect: Connection refused (error 111)
05:26:21.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:21.751729 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 35555 } attempt: 3
05:26:21.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:21.755573 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:35555: Network error: Client connection negotiation failed: client connection to 127.0.93.189:35555: connect: Connection refused (error 111)
05:26:21.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:21.800415 456 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:26:21.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:21.800946 456 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:26:21.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:21.801568 456 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:26:21.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:21.832326 456 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:26:21.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:21.832649 456 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:26:21.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:21.832876 456 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:26:21.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:21.833061 456 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:26:21.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:21.833232 456 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:26:21.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:21.833444 456 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34201/jwks.json
05:26:21.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:21.868391 456 master_runner.cc:386] Master server non-default flags:
05:26:21.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:26:21.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:46351
05:26:21.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:26:21.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:26:21.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-1/data
05:26:21.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-1/wal
05:26:21.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:26:21.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:42033,127.0.93.189:35555,127.0.93.188:41061
05:26:21.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:26:21.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:26:21.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:26:21.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.189:35555
05:26:21.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:26:21.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:26:21.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:26:21.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34201/jwks.json
05:26:21.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:26:21.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:26:21.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-1/data/info.pb
05:26:21.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.189
05:26:21.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:26:21.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:26:21.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:26:21.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:26:21.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:26:21.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:26:21.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-1/logs
05:26:21.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:26:21.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:26:21.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:26:21.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:26:21.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:26:21.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:26:21.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:26:21.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 04:01:01 UTC on ccycloud.quasar-phqekz.root.comops.site
05:26:21.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679192
05:26:21.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:26:21.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:21.869580 456 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:26:21.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:21.871693 456 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:26:21.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:21.889513 466 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:26:21.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:21.899010 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 35555 } attempt: 4
05:26:21.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:21.903633 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:35555: Network error: Client connection negotiation failed: client connection to 127.0.93.189:35555: connect: Connection refused (error 111)
05:26:22.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:22.203092 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 35555 } attempt: 5
05:26:22.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:22.207712 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:35555: Network error: Client connection negotiation failed: client connection to 127.0.93.189:35555: connect: Connection refused (error 111)
05:26:22.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:22.755137 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 35555 } attempt: 6
05:26:22.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:22.762723 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:35555: Network error: Client connection negotiation failed: client connection to 127.0.93.189:35555: connect: Connection refused (error 111)
05:26:23.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:21.889928 465 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:26:23.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:21.890084 468 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:26:23.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:23.089097 467 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
05:26:23.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:23.089378 456 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
05:26:23.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:23.093760 456 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:26:23.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:23.097213 456 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:26:23.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:23.100787 456 hybrid_clock.cc:648] HybridClock initialized: now 1790659583100416 us; error 383 us; skew 500 ppm
05:26:23.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:23.101552 456 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:26:23.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:23.112242 456 webserver.cc:460] Webserver started at http://127.0.93.189:43871/ using document root <none> and password file <none>
05:26:23.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:23.113199 456 fs_manager.cc:359] Metadata directory not provided
05:26:23.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:23.113390 456 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:26:23.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:23.113761 456 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:26:23.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:23.118788 456 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-1/data/instance:
05:26:23.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6dc601125bc74bcea9e2b96c10d59a74"
05:26:23.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:26:23 on dist-test-slave-gvjk"
05:26:23.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:23.120071 456 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-1/wal/instance:
05:26:23.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6dc601125bc74bcea9e2b96c10d59a74"
05:26:23.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:26:23 on dist-test-slave-gvjk"
05:26:23.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:23.127894 456 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
05:26:23.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:23.133502 477 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:26:23.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:23.134640 456 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
05:26:23.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:23.134904 456 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-1/data,/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-1/wal
05:26:23.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6dc601125bc74bcea9e2b96c10d59a74"
05:26:23.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:26:23 on dist-test-slave-gvjk"
05:26:23.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:23.135185 456 fs_report.cc:383] FS layout report
05:26:23.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:26:23.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-1/wal
05:26:23.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-1/wal
05:26:23.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-1/data/data
05:26:23.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:26:23.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:26:23.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:26:23.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:26:23.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:26:23.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:26:23.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:26:23.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:26:23.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:26:23.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:26:23.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:26:23.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:23.194737 456 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:26:23.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:23.196563 456 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:26:23.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:23.196944 456 kserver.cc:163] Server-wide thread pool size limit: 3276
05:26:23.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:23.270401 456 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.189:35555
05:26:23.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:23.270512 528 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.189:35555 every 8 connection(s)
05:26:23.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:23.273089 456 server_base.cc:1166] Dumped server information to /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-1/data/info.pb
05:26:23.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:23.275938 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-task_eGFIx/build/tsan/bin/kudu as pid 456
05:26:23.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:23.276480 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-1/wal/instance
05:26:23.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:23.278488 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-task_eGFIx/build/tsan/bin/kudu
05:26:23.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_eGFIx/build/tsan/bin/kudu
05:26:23.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-2/wal
05:26:23.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-2/data
05:26:23.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:26:23.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:26:23.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:26:23.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:26:23.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:26:23.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:26:23.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-2/logs
05:26:23.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-2/data/info.pb
05:26:23.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:26:23.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:26:23.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:26:23.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:26:23.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:26:23.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:26:23.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:26:23.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:26:23.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:26:23.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:26:23.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:26:23.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:26:23.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.188:41061
05:26:23.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.188
05:26:23.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:26:23.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:46351
05:26:23.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:26:23.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:26:23.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:26:23.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:26:23.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:42033,127.0.93.189:35555,127.0.93.188:41061
05:26:23.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:26:23.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34201/jwks.json with env {}
05:26:23.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:23.278669 529 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:26:23.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:23.292989 529 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42033 } has no permanent_uuid. Determining permanent_uuid...
05:26:23.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:23.316782 529 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.189" port: 35555 } has no permanent_uuid. Determining permanent_uuid...
05:26:23.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:23.332858 529 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41061 } has no permanent_uuid. Determining permanent_uuid...
05:26:23.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:23.335397 478 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.93.188:41061: connect: Connection refused (error 111)
05:26:23.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:23.339702 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:41061: Network error: Client connection negotiation failed: client connection to 127.0.93.188:41061: connect: Connection refused (error 111)
05:26:23.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:23.389182 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41061 } attempt: 1
05:26:23.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:23.393409 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:41061: Network error: Client connection negotiation failed: client connection to 127.0.93.188:41061: connect: Connection refused (error 111)
05:26:23.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:23.461757 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41061 } attempt: 2
05:26:23.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:23.465764 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:41061: Network error: Client connection negotiation failed: client connection to 127.0.93.188:41061: connect: Connection refused (error 111)
05:26:23.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:23.557139 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41061 } attempt: 3
05:26:23.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:23.560861 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:41061: Network error: Client connection negotiation failed: client connection to 127.0.93.188:41061: connect: Connection refused (error 111)
05:26:23.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:23.603226 531 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:26:23.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:23.603946 531 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:26:23.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:23.604628 531 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:26:23.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:23.634812 531 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:26:23.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:23.635169 531 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:26:23.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:23.635500 531 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:26:23.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:23.635850 531 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:26:23.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:23.636188 531 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:26:23.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:23.636407 531 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34201/jwks.json
05:26:23.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:23.674017 531 master_runner.cc:386] Master server non-default flags:
05:26:23.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:26:23.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:46351
05:26:23.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:26:23.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:26:23.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-2/data
05:26:23.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-2/wal
05:26:23.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:26:23.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:42033,127.0.93.189:35555,127.0.93.188:41061
05:26:23.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:26:23.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:26:23.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:26:23.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.188:41061
05:26:23.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:26:23.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:26:23.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:26:23.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34201/jwks.json
05:26:23.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:26:23.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:26:23.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-2/data/info.pb
05:26:23.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.188
05:26:23.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:26:23.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:26:23.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:26:23.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:26:23.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:26:23.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:26:23.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-2/logs
05:26:23.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:26:23.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:26:23.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:26:23.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:26:23.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:26:23.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:26:23.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:26:23.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 04:01:01 UTC on ccycloud.quasar-phqekz.root.comops.site
05:26:23.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679192
05:26:23.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:26:23.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:23.675496 531 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:26:23.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:23.677486 531 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:26:23.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:23.697561 541 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:26:23.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:23.704676 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41061 } attempt: 4
05:26:23.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:23.708766 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:41061: Network error: Client connection negotiation failed: client connection to 127.0.93.188:41061: connect: Connection refused (error 111)
05:26:23.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:23.823093 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 35555 } attempt: 7
05:26:23.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:23.838377 454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41061 } has no permanent_uuid. Determining permanent_uuid...
05:26:23.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:23.843343 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:41061: Network error: Client connection negotiation failed: client connection to 127.0.93.188:41061: connect: Connection refused (error 111)
05:26:23.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:23.901925 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41061 } attempt: 1
05:26:23.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:23.905718 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:41061: Network error: Client connection negotiation failed: client connection to 127.0.93.188:41061: connect: Connection refused (error 111)
05:26:23.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:23.987111 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41061 } attempt: 2
05:26:23.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:23.991731 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:41061: Network error: Client connection negotiation failed: client connection to 127.0.93.188:41061: connect: Connection refused (error 111)
05:26:24.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:24.008152 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41061 } attempt: 5
05:26:24.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:24.013147 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:41061: Network error: Client connection negotiation failed: client connection to 127.0.93.188:41061: connect: Connection refused (error 111)
05:26:24.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:24.077170 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41061 } attempt: 3
05:26:24.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:24.081403 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:41061: Network error: Client connection negotiation failed: client connection to 127.0.93.188:41061: connect: Connection refused (error 111)
05:26:24.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:24.221786 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41061 } attempt: 4
05:26:24.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:24.226115 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:41061: Network error: Client connection negotiation failed: client connection to 127.0.93.188:41061: connect: Connection refused (error 111)
05:26:24.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:24.509531 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41061 } attempt: 5
05:26:24.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:24.514285 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:41061: Network error: Client connection negotiation failed: client connection to 127.0.93.188:41061: connect: Connection refused (error 111)
05:26:24.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:24.560544 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41061 } attempt: 6
05:26:24.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:24.568569 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:41061: Network error: Client connection negotiation failed: client connection to 127.0.93.188:41061: connect: Connection refused (error 111)
05:26:24.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:23.697594 542 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:26:24.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:23.698799 544 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:26:24.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:24.921973 543 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
05:26:24.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:24.922114 531 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
05:26:24.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:24.929174 531 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:26:24.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:24.932762 531 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:26:24.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:24.934211 531 hybrid_clock.cc:648] HybridClock initialized: now 1790659584934171 us; error 41 us; skew 500 ppm
05:26:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:24.935006 531 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:26:24.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:24.943147 531 webserver.cc:460] Webserver started at http://127.0.93.188:34413/ using document root <none> and password file <none>
05:26:24.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:24.944182 531 fs_manager.cc:359] Metadata directory not provided
05:26:24.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:24.944577 531 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:26:24.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:24.945044 531 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:26:24.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:24.951385 531 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-2/data/instance:
05:26:24.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e9ca8f08944c42e991817e7d559f6114"
05:26:24.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:26:24 on dist-test-slave-gvjk"
05:26:24.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:24.952621 531 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-2/wal/instance:
05:26:24.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e9ca8f08944c42e991817e7d559f6114"
05:26:24.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:26:24 on dist-test-slave-gvjk"
05:26:24.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:24.960250 531 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.003s sys 0.005s
05:26:24.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:24.966073 555 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:26:24.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:24.967231 531 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
05:26:24.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:24.967523 531 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-2/data,/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-2/wal
05:26:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e9ca8f08944c42e991817e7d559f6114"
05:26:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:26:24 on dist-test-slave-gvjk"
05:26:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:24.967905 531 fs_report.cc:383] FS layout report
05:26:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:26:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-2/wal
05:26:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-2/wal
05:26:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-2/data/data
05:26:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:26:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:26:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:26:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:26:24.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:26:24.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:26:24.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:26:24.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:26:24.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:26:24.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:26:24.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:26:25.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.023151 531 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:26:25.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.024672 531 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:26:25.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.025094 531 kserver.cc:163] Server-wide thread pool size limit: 3276
05:26:25.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.067131 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41061 } attempt: 6
05:26:25.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:25.078270 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:41061: Network error: Client connection negotiation failed: client connection to 127.0.93.188:41061: connect: Connection refused (error 111)
05:26:25.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.105757 531 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.188:41061
05:26:25.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.105847 608 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.188:41061 every 8 connection(s)
05:26:25.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.108335 531 server_base.cc:1166] Dumped server information to /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-2/data/info.pb
05:26:25.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.113473 609 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:26:25.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.114122 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-task_eGFIx/build/tsan/bin/kudu as pid 531
05:26:25.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.114425 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/master-2/wal/instance
05:26:25.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.131047 609 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42033 } has no permanent_uuid. Determining permanent_uuid...
05:26:25.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.160094 609 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.189" port: 35555 } has no permanent_uuid. Determining permanent_uuid...
05:26:25.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.172742 609 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41061 } has no permanent_uuid. Determining permanent_uuid...
05:26:25.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.190024 609 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e9ca8f08944c42e991817e7d559f6114: Bootstrap starting.
05:26:25.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.197098 609 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P e9ca8f08944c42e991817e7d559f6114: Neither blocks nor log segments found. Creating new log.
05:26:25.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.198837 609 log.cc:824] T 00000000000000000000000000000000 P e9ca8f08944c42e991817e7d559f6114: Log is configured to *not* fsync() on all Append() calls
05:26:25.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.204257 609 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e9ca8f08944c42e991817e7d559f6114: No bootstrap required, opened a new log
05:26:25.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.225634 609 raft_consensus.cc:348] T 00000000000000000000000000000000 P e9ca8f08944c42e991817e7d559f6114 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5f93bbf8c6e414490f1b5e236d7830f" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42033 } } peers { permanent_uuid: "6dc601125bc74bcea9e2b96c10d59a74" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 35555 } } peers { permanent_uuid: "e9ca8f08944c42e991817e7d559f6114" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41061 } }
05:26:25.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.226643 609 raft_consensus.cc:374] T 00000000000000000000000000000000 P e9ca8f08944c42e991817e7d559f6114 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
05:26:25.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.226912 609 raft_consensus.cc:729] T 00000000000000000000000000000000 P e9ca8f08944c42e991817e7d559f6114 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e9ca8f08944c42e991817e7d559f6114, State: Initialized, Role: FOLLOWER
05:26:25.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.227746 609 consensus_queue.cc:260] T 00000000000000000000000000000000 P e9ca8f08944c42e991817e7d559f6114 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5f93bbf8c6e414490f1b5e236d7830f" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42033 } } peers { permanent_uuid: "6dc601125bc74bcea9e2b96c10d59a74" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 35555 } } peers { permanent_uuid: "e9ca8f08944c42e991817e7d559f6114" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41061 } }
05:26:25.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.230234 618 sys_catalog.cc:455] T 00000000000000000000000000000000 P e9ca8f08944c42e991817e7d559f6114 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5f93bbf8c6e414490f1b5e236d7830f" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42033 } } peers { permanent_uuid: "6dc601125bc74bcea9e2b96c10d59a74" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 35555 } } peers { permanent_uuid: "e9ca8f08944c42e991817e7d559f6114" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41061 } } }
05:26:25.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.230886 618 sys_catalog.cc:458] T 00000000000000000000000000000000 P e9ca8f08944c42e991817e7d559f6114 [sys.catalog]: This master's current role is: FOLLOWER
05:26:25.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.231976 609 sys_catalog.cc:564] T 00000000000000000000000000000000 P e9ca8f08944c42e991817e7d559f6114 [sys.catalog]: configured and running, proceeding with master startup.
05:26:25.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:25.257588 629 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e9ca8f08944c42e991817e7d559f6114: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:26:25.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:25.257889 629 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:26:25.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:25.280313 561 tablet.cc:2369] T 00000000000000000000000000000000 P e9ca8f08944c42e991817e7d559f6114: Can't schedule compaction. Clean time has not been advanced past its initial value.
05:26:25.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.253345 531 master_runner.cc:186] Error getting master registration for 127.0.93.190:42033: OK, instance_id { permanent_uuid: "e5f93bbf8c6e414490f1b5e236d7830f" instance_seqno: 1790659581457260 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
05:26:25.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.285246 531 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:26:25.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.453917 618 raft_consensus.cc:482] T 00000000000000000000000000000000 P e9ca8f08944c42e991817e7d559f6114 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:26:25.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.454459 618 raft_consensus.cc:504] T 00000000000000000000000000000000 P e9ca8f08944c42e991817e7d559f6114 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5f93bbf8c6e414490f1b5e236d7830f" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42033 } } peers { permanent_uuid: "6dc601125bc74bcea9e2b96c10d59a74" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 35555 } } peers { permanent_uuid: "e9ca8f08944c42e991817e7d559f6114" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41061 } }
05:26:25.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.456679 618 leader_election.cc:288] T 00000000000000000000000000000000 P e9ca8f08944c42e991817e7d559f6114 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e5f93bbf8c6e414490f1b5e236d7830f (127.0.93.190:42033), 6dc601125bc74bcea9e2b96c10d59a74 (127.0.93.189:35555)
05:26:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.629024 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41061 } attempt: 7
05:26:25.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.651647 529 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6dc601125bc74bcea9e2b96c10d59a74: Bootstrap starting.
05:26:25.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.659333 529 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 6dc601125bc74bcea9e2b96c10d59a74: Neither blocks nor log segments found. Creating new log.
05:26:25.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.661518 529 log.cc:824] T 00000000000000000000000000000000 P 6dc601125bc74bcea9e2b96c10d59a74: Log is configured to *not* fsync() on all Append() calls
05:26:25.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.673079 529 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6dc601125bc74bcea9e2b96c10d59a74: No bootstrap required, opened a new log
05:26:25.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.707536 529 raft_consensus.cc:348] T 00000000000000000000000000000000 P 6dc601125bc74bcea9e2b96c10d59a74 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5f93bbf8c6e414490f1b5e236d7830f" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42033 } } peers { permanent_uuid: "6dc601125bc74bcea9e2b96c10d59a74" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 35555 } } peers { permanent_uuid: "e9ca8f08944c42e991817e7d559f6114" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41061 } }
05:26:25.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.708488 529 raft_consensus.cc:374] T 00000000000000000000000000000000 P 6dc601125bc74bcea9e2b96c10d59a74 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
05:26:25.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.708806 529 raft_consensus.cc:729] T 00000000000000000000000000000000 P 6dc601125bc74bcea9e2b96c10d59a74 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6dc601125bc74bcea9e2b96c10d59a74, State: Initialized, Role: FOLLOWER
05:26:25.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.709879 529 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6dc601125bc74bcea9e2b96c10d59a74 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5f93bbf8c6e414490f1b5e236d7830f" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42033 } } peers { permanent_uuid: "6dc601125bc74bcea9e2b96c10d59a74" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 35555 } } peers { permanent_uuid: "e9ca8f08944c42e991817e7d559f6114" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41061 } }
05:26:25.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.712673 633 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6dc601125bc74bcea9e2b96c10d59a74 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5f93bbf8c6e414490f1b5e236d7830f" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42033 } } peers { permanent_uuid: "6dc601125bc74bcea9e2b96c10d59a74" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 35555 } } peers { permanent_uuid: "e9ca8f08944c42e991817e7d559f6114" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41061 } } }
05:26:25.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.713642 633 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6dc601125bc74bcea9e2b96c10d59a74 [sys.catalog]: This master's current role is: FOLLOWER
05:26:25.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.715134 529 sys_catalog.cc:564] T 00000000000000000000000000000000 P 6dc601125bc74bcea9e2b96c10d59a74 [sys.catalog]: configured and running, proceeding with master startup.
05:26:25.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.715272 504 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e9ca8f08944c42e991817e7d559f6114" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6dc601125bc74bcea9e2b96c10d59a74" is_pre_election: true
05:26:25.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.716179 504 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 6dc601125bc74bcea9e2b96c10d59a74 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e9ca8f08944c42e991817e7d559f6114 in term 0.
05:26:25.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.717480 559 leader_election.cc:302] T 00000000000000000000000000000000 P e9ca8f08944c42e991817e7d559f6114 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6dc601125bc74bcea9e2b96c10d59a74, e9ca8f08944c42e991817e7d559f6114; no voters:
05:26:25.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.718379 618 raft_consensus.cc:2781] T 00000000000000000000000000000000 P e9ca8f08944c42e991817e7d559f6114 [term 0 FOLLOWER]: Leader pre-election won for term 1
05:26:25.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.719259 618 raft_consensus.cc:482] T 00000000000000000000000000000000 P e9ca8f08944c42e991817e7d559f6114 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
05:26:25.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.719723 618 raft_consensus.cc:3037] T 00000000000000000000000000000000 P e9ca8f08944c42e991817e7d559f6114 [term 0 FOLLOWER]: Advancing to term 1
05:26:25.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.736366 618 raft_consensus.cc:504] T 00000000000000000000000000000000 P e9ca8f08944c42e991817e7d559f6114 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5f93bbf8c6e414490f1b5e236d7830f" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42033 } } peers { permanent_uuid: "6dc601125bc74bcea9e2b96c10d59a74" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 35555 } } peers { permanent_uuid: "e9ca8f08944c42e991817e7d559f6114" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41061 } }
05:26:25.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.738727 618 leader_election.cc:288] T 00000000000000000000000000000000 P e9ca8f08944c42e991817e7d559f6114 [CANDIDATE]: Term 1 election: Requested vote from peers e5f93bbf8c6e414490f1b5e236d7830f (127.0.93.190:42033), 6dc601125bc74bcea9e2b96c10d59a74 (127.0.93.189:35555)
05:26:25.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.741230 504 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e9ca8f08944c42e991817e7d559f6114" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6dc601125bc74bcea9e2b96c10d59a74"
05:26:25.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.741966 504 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 6dc601125bc74bcea9e2b96c10d59a74 [term 0 FOLLOWER]: Advancing to term 1
05:26:25.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.755615 504 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 6dc601125bc74bcea9e2b96c10d59a74 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e9ca8f08944c42e991817e7d559f6114 in term 1.
05:26:25.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.756635 559 leader_election.cc:302] T 00000000000000000000000000000000 P e9ca8f08944c42e991817e7d559f6114 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6dc601125bc74bcea9e2b96c10d59a74, e9ca8f08944c42e991817e7d559f6114; no voters:
05:26:25.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.757411 618 raft_consensus.cc:2781] T 00000000000000000000000000000000 P e9ca8f08944c42e991817e7d559f6114 [term 1 FOLLOWER]: Leader election won for term 1
05:26:25.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.773205 618 raft_consensus.cc:686] T 00000000000000000000000000000000 P e9ca8f08944c42e991817e7d559f6114 [term 1 LEADER]: Becoming Leader. State: Replica: e9ca8f08944c42e991817e7d559f6114, State: Running, Role: LEADER
05:26:25.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.774875 618 consensus_queue.cc:237] T 00000000000000000000000000000000 P e9ca8f08944c42e991817e7d559f6114 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5f93bbf8c6e414490f1b5e236d7830f" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42033 } } peers { permanent_uuid: "6dc601125bc74bcea9e2b96c10d59a74" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 35555 } } peers { permanent_uuid: "e9ca8f08944c42e991817e7d559f6114" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41061 } }
05:26:25.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.783293 639 sys_catalog.cc:455] T 00000000000000000000000000000000 P e9ca8f08944c42e991817e7d559f6114 [sys.catalog]: SysCatalogTable state changed. Reason: New leader e9ca8f08944c42e991817e7d559f6114. Latest consensus state: current_term: 1 leader_uuid: "e9ca8f08944c42e991817e7d559f6114" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5f93bbf8c6e414490f1b5e236d7830f" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42033 } } peers { permanent_uuid: "6dc601125bc74bcea9e2b96c10d59a74" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 35555 } } peers { permanent_uuid: "e9ca8f08944c42e991817e7d559f6114" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41061 } } }
05:26:25.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.784499 639 sys_catalog.cc:458] T 00000000000000000000000000000000 P e9ca8f08944c42e991817e7d559f6114 [sys.catalog]: This master's current role is: LEADER
05:26:25.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.792881 646 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
05:26:25.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.798808 647 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
05:26:25.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.801604 646 catalog_manager.cc:1484] Initializing Kudu cluster ID...
05:26:25.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.821928 504 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 6dc601125bc74bcea9e2b96c10d59a74 [term 1 FOLLOWER]: Refusing update from remote peer e9ca8f08944c42e991817e7d559f6114: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
05:26:25.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.823647 618 consensus_queue.cc:1035] T 00000000000000000000000000000000 P e9ca8f08944c42e991817e7d559f6114 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6dc601125bc74bcea9e2b96c10d59a74" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 35555 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
05:26:25.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.802618 456 master_runner.cc:186] Error getting master registration for 127.0.93.190:42033: OK, instance_id { permanent_uuid: "e5f93bbf8c6e414490f1b5e236d7830f" instance_seqno: 1790659581457260 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
05:26:25.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.835055 456 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:26:25.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.853581 648 mvcc.cc:205] Tried to move back new op lower bound from 7334541663506706432 to 7334541663345963008. Current Snapshot: MvccSnapshot[applied={T|T < 7334541663506706432}]
05:26:25.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.854856 633 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6dc601125bc74bcea9e2b96c10d59a74 [sys.catalog]: SysCatalogTable state changed. Reason: New leader e9ca8f08944c42e991817e7d559f6114. Latest consensus state: current_term: 1 leader_uuid: "e9ca8f08944c42e991817e7d559f6114" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5f93bbf8c6e414490f1b5e236d7830f" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42033 } } peers { permanent_uuid: "6dc601125bc74bcea9e2b96c10d59a74" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 35555 } } peers { permanent_uuid: "e9ca8f08944c42e991817e7d559f6114" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41061 } } }
05:26:25.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.855958 633 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6dc601125bc74bcea9e2b96c10d59a74 [sys.catalog]: This master's current role is: FOLLOWER
05:26:25.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.865957 618 sys_catalog.cc:455] T 00000000000000000000000000000000 P e9ca8f08944c42e991817e7d559f6114 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "e9ca8f08944c42e991817e7d559f6114" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5f93bbf8c6e414490f1b5e236d7830f" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42033 } } peers { permanent_uuid: "6dc601125bc74bcea9e2b96c10d59a74" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 35555 } } peers { permanent_uuid: "e9ca8f08944c42e991817e7d559f6114" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41061 } } }
05:26:25.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.866467 646 catalog_manager.cc:1347] Generated new cluster ID: 58d9850068a842698e8a26c762b0c9ae
05:26:25.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.866693 618 sys_catalog.cc:458] T 00000000000000000000000000000000 P e9ca8f08944c42e991817e7d559f6114 [sys.catalog]: This master's current role is: LEADER
05:26:25.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.866773 646 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
05:26:25.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.885336 652 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6dc601125bc74bcea9e2b96c10d59a74 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "e9ca8f08944c42e991817e7d559f6114" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5f93bbf8c6e414490f1b5e236d7830f" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42033 } } peers { permanent_uuid: "6dc601125bc74bcea9e2b96c10d59a74" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 35555 } } peers { permanent_uuid: "e9ca8f08944c42e991817e7d559f6114" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41061 } } }
05:26:25.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.886590 652 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6dc601125bc74bcea9e2b96c10d59a74 [sys.catalog]: This master's current role is: FOLLOWER
05:26:25.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.892503 646 catalog_manager.cc:1370] Generated new certificate authority record
05:26:25.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.895433 646 catalog_manager.cc:1504] Loading token signing keys...
05:26:25.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.922086 646 catalog_manager.cc:5868] T 00000000000000000000000000000000 P e9ca8f08944c42e991817e7d559f6114: Generated new TSK 0
05:26:25.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.924043 646 catalog_manager.cc:1514] Initializing in-progress tserver states...
05:26:25.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:25.964174 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-task_eGFIx/build/tsan/bin/kudu
05:26:25.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_eGFIx/build/tsan/bin/kudu
05:26:25.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-0/wal
05:26:25.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-0/data
05:26:25.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:26:25.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:26:25.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:26:25.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:26:25.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:26:25.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:26:25.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-0/logs
05:26:25.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-0/data/info.pb
05:26:25.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:26:25.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:26:25.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:26:25.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:26:25.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:26:25.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:26:25.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:26:25.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:26:25.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
05:26:25.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:26:25.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.129:0
05:26:25.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.129
05:26:25.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.129
05:26:25.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:26:25.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:42033,127.0.93.189:35555,127.0.93.188:41061
05:26:25.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:46351
05:26:25.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:26:25.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:26:25.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
05:26:26.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:26.111670 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41061 } attempt: 7
05:26:26.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:26.150763 454 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e5f93bbf8c6e414490f1b5e236d7830f: Bootstrap starting.
05:26:26.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:26.160461 454 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P e5f93bbf8c6e414490f1b5e236d7830f: Neither blocks nor log segments found. Creating new log.
05:26:26.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:26.163546 454 log.cc:824] T 00000000000000000000000000000000 P e5f93bbf8c6e414490f1b5e236d7830f: Log is configured to *not* fsync() on all Append() calls
05:26:26.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:26.190688 454 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e5f93bbf8c6e414490f1b5e236d7830f: No bootstrap required, opened a new log
05:26:26.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:26.226051 454 raft_consensus.cc:348] T 00000000000000000000000000000000 P e5f93bbf8c6e414490f1b5e236d7830f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5f93bbf8c6e414490f1b5e236d7830f" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42033 } } peers { permanent_uuid: "6dc601125bc74bcea9e2b96c10d59a74" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 35555 } } peers { permanent_uuid: "e9ca8f08944c42e991817e7d559f6114" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41061 } }
05:26:26.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:26.227232 454 raft_consensus.cc:374] T 00000000000000000000000000000000 P e5f93bbf8c6e414490f1b5e236d7830f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
05:26:26.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:26.227694 454 raft_consensus.cc:729] T 00000000000000000000000000000000 P e5f93bbf8c6e414490f1b5e236d7830f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e5f93bbf8c6e414490f1b5e236d7830f, State: Initialized, Role: FOLLOWER
05:26:26.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:26.228854 454 consensus_queue.cc:260] T 00000000000000000000000000000000 P e5f93bbf8c6e414490f1b5e236d7830f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5f93bbf8c6e414490f1b5e236d7830f" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42033 } } peers { permanent_uuid: "6dc601125bc74bcea9e2b96c10d59a74" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 35555 } } peers { permanent_uuid: "e9ca8f08944c42e991817e7d559f6114" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41061 } }
05:26:26.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:26.237165 657 sys_catalog.cc:455] T 00000000000000000000000000000000 P e5f93bbf8c6e414490f1b5e236d7830f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5f93bbf8c6e414490f1b5e236d7830f" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42033 } } peers { permanent_uuid: "6dc601125bc74bcea9e2b96c10d59a74" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 35555 } } peers { permanent_uuid: "e9ca8f08944c42e991817e7d559f6114" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41061 } } }
05:26:26.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:26.241994 657 sys_catalog.cc:458] T 00000000000000000000000000000000 P e5f93bbf8c6e414490f1b5e236d7830f [sys.catalog]: This master's current role is: FOLLOWER
05:26:26.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:26.241663 454 sys_catalog.cc:564] T 00000000000000000000000000000000 P e5f93bbf8c6e414490f1b5e236d7830f [sys.catalog]: configured and running, proceeding with master startup.
05:26:26.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:26.243027 427 raft_consensus.cc:3037] T 00000000000000000000000000000000 P e5f93bbf8c6e414490f1b5e236d7830f [term 0 FOLLOWER]: Advancing to term 1
05:26:26.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:26.241824 429 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e9ca8f08944c42e991817e7d559f6114" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e5f93bbf8c6e414490f1b5e236d7830f" is_pre_election: true
05:26:26.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:26.244041 428 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e9ca8f08944c42e991817e7d559f6114" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e5f93bbf8c6e414490f1b5e236d7830f"
05:26:26.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:26.255415 427 raft_consensus.cc:1261] T 00000000000000000000000000000000 P e5f93bbf8c6e414490f1b5e236d7830f [term 1 FOLLOWER]: Refusing update from remote peer e9ca8f08944c42e991817e7d559f6114: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
05:26:26.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:26.257900 639 consensus_queue.cc:1035] T 00000000000000000000000000000000 P e9ca8f08944c42e991817e7d559f6114 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e5f93bbf8c6e414490f1b5e236d7830f" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42033 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
05:26:26.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:26.260013 657 raft_consensus.cc:482] T 00000000000000000000000000000000 P e5f93bbf8c6e414490f1b5e236d7830f [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:26:26.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:26.260465 657 raft_consensus.cc:504] T 00000000000000000000000000000000 P e5f93bbf8c6e414490f1b5e236d7830f [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5f93bbf8c6e414490f1b5e236d7830f" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42033 } } peers { permanent_uuid: "6dc601125bc74bcea9e2b96c10d59a74" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 35555 } } peers { permanent_uuid: "e9ca8f08944c42e991817e7d559f6114" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41061 } }
05:26:26.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:26.262742 657 leader_election.cc:288] T 00000000000000000000000000000000 P e5f93bbf8c6e414490f1b5e236d7830f [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 6dc601125bc74bcea9e2b96c10d59a74 (127.0.93.189:35555), e9ca8f08944c42e991817e7d559f6114 (127.0.93.188:41061)
05:26:26.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:26.265807 503 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e5f93bbf8c6e414490f1b5e236d7830f" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6dc601125bc74bcea9e2b96c10d59a74" is_pre_election: true
05:26:26.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:26.266393 582 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e5f93bbf8c6e414490f1b5e236d7830f" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e9ca8f08944c42e991817e7d559f6114" is_pre_election: true
05:26:26.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:26.268440 406 leader_election.cc:302] T 00000000000000000000000000000000 P e5f93bbf8c6e414490f1b5e236d7830f [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e5f93bbf8c6e414490f1b5e236d7830f; no voters: 6dc601125bc74bcea9e2b96c10d59a74, e9ca8f08944c42e991817e7d559f6114
05:26:26.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:26.328462 661 raft_consensus.cc:2726] T 00000000000000000000000000000000 P e5f93bbf8c6e414490f1b5e236d7830f [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
05:26:26.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:26.332945 657 sys_catalog.cc:455] T 00000000000000000000000000000000 P e5f93bbf8c6e414490f1b5e236d7830f [sys.catalog]: SysCatalogTable state changed. Reason: New leader e9ca8f08944c42e991817e7d559f6114. Latest consensus state: current_term: 1 leader_uuid: "e9ca8f08944c42e991817e7d559f6114" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5f93bbf8c6e414490f1b5e236d7830f" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42033 } } peers { permanent_uuid: "6dc601125bc74bcea9e2b96c10d59a74" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 35555 } } peers { permanent_uuid: "e9ca8f08944c42e991817e7d559f6114" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41061 } } }
05:26:26.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:26.333474 666 sys_catalog.cc:455] T 00000000000000000000000000000000 P e5f93bbf8c6e414490f1b5e236d7830f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "e9ca8f08944c42e991817e7d559f6114" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5f93bbf8c6e414490f1b5e236d7830f" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42033 } } peers { permanent_uuid: "6dc601125bc74bcea9e2b96c10d59a74" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 35555 } } peers { permanent_uuid: "e9ca8f08944c42e991817e7d559f6114" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41061 } } }
05:26:26.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:26.334542 666 sys_catalog.cc:458] T 00000000000000000000000000000000 P e5f93bbf8c6e414490f1b5e236d7830f [sys.catalog]: This master's current role is: FOLLOWER
05:26:26.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:26.334113 657 sys_catalog.cc:458] T 00000000000000000000000000000000 P e5f93bbf8c6e414490f1b5e236d7830f [sys.catalog]: This master's current role is: FOLLOWER
05:26:26.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:26.368784 675 catalog_manager.cc:1259] Loaded cluster ID: 58d9850068a842698e8a26c762b0c9ae
05:26:26.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:26.369390 675 catalog_manager.cc:1552] T 00000000000000000000000000000000 P e5f93bbf8c6e414490f1b5e236d7830f: loading cluster ID for follower catalog manager: success
05:26:26.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:26.379182 675 catalog_manager.cc:1574] T 00000000000000000000000000000000 P e5f93bbf8c6e414490f1b5e236d7830f: acquiring CA information for follower catalog manager: success
05:26:26.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:26.385196 675 catalog_manager.cc:1602] T 00000000000000000000000000000000 P e5f93bbf8c6e414490f1b5e236d7830f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
05:26:26.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:26.511188 654 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:26:26.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:26.512171 654 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:26:26.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:26.513124 654 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:26:26.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:26.570778 654 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:26:26.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:26.572127 654 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.93.129
05:26:26.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:26.631817 654 tablet_server_runner.cc:78] Tablet server non-default flags:
05:26:26.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:26:26.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:46351
05:26:26.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:26:26.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:26:26.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-0/data
05:26:26.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-0/wal
05:26:26.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:26:26.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:26:26.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.129:0
05:26:26.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:26:26.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:26:26.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:26:26.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-0/data/info.pb
05:26:26.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.129
05:26:26.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:26:26.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:42033,127.0.93.189:35555,127.0.93.188:41061
05:26:26.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:26:26.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:26:26.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:26:26.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:26:26.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:26:26.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.129
05:26:26.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-0/logs
05:26:26.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:26:26.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:26:26.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:26:26.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
05:26:26.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:26:26.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:26:26.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:26:26.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 04:01:01 UTC on ccycloud.quasar-phqekz.root.comops.site
05:26:26.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679192
05:26:26.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:26:26.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:26.634542 654 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:26:26.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:26.636691 654 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:26:26.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:26.660634 682 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:26:26.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:26.701586 639 sys_catalog.cc:455] T 00000000000000000000000000000000 P e9ca8f08944c42e991817e7d559f6114 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "e9ca8f08944c42e991817e7d559f6114" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5f93bbf8c6e414490f1b5e236d7830f" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42033 } } peers { permanent_uuid: "6dc601125bc74bcea9e2b96c10d59a74" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 35555 } } peers { permanent_uuid: "e9ca8f08944c42e991817e7d559f6114" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41061 } } }
05:26:26.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:26.702355 639 sys_catalog.cc:458] T 00000000000000000000000000000000 P e9ca8f08944c42e991817e7d559f6114 [sys.catalog]: This master's current role is: LEADER
05:26:26.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:26.825784 647 catalog_manager.cc:1259] Loaded cluster ID: 58d9850068a842698e8a26c762b0c9ae
05:26:26.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:26.831763 647 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 6dc601125bc74bcea9e2b96c10d59a74: loading cluster ID for follower catalog manager: success
05:26:26.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:26.853801 647 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 6dc601125bc74bcea9e2b96c10d59a74: acquiring CA information for follower catalog manager: success
05:26:26.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:26.869652 647 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 6dc601125bc74bcea9e2b96c10d59a74: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
05:26:29.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:28.056022 681 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 654
05:26:29.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:29.603015 654 thread.cc:636] GCE (cloud detector) Time spent creating pthread: real 2.943s user 0.936s sys 1.546s
05:26:29.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:29.603324 654 thread.cc:602] GCE (cloud detector) Time spent starting thread: real 2.944s user 0.936s sys 1.546s
05:26:29.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:26.664976 683 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:26:29.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:29.612785 654 server_base.cc:1034] running on GCE node
05:26:29.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:29.612993 691 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:26:29.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:29.614233 654 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:26:29.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:29.617028 654 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:26:29.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:29.618515 654 hybrid_clock.cc:648] HybridClock initialized: now 1790659589618445 us; error 76 us; skew 500 ppm
05:26:29.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:29.619415 654 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:26:29.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:29.626339 654 webserver.cc:460] Webserver started at http://127.0.93.129:38715/ using document root <none> and password file <none>
05:26:29.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:29.627290 654 fs_manager.cc:359] Metadata directory not provided
05:26:29.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:29.627532 654 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:26:29.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:29.628067 654 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:26:29.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:29.634147 654 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-0/data/instance:
05:26:29.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "07491faa591e450aaa7de53fae364b9e"
05:26:29.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:26:29 on dist-test-slave-gvjk"
05:26:29.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:29.635349 654 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-0/wal/instance:
05:26:29.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "07491faa591e450aaa7de53fae364b9e"
05:26:29.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:26:29 on dist-test-slave-gvjk"
05:26:29.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:29.642846 654 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.001s
05:26:29.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:29.648952 698 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:26:29.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:29.650235 654 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
05:26:29.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:29.650584 654 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-0/data,/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-0/wal
05:26:29.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "07491faa591e450aaa7de53fae364b9e"
05:26:29.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:26:29 on dist-test-slave-gvjk"
05:26:29.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:29.650949 654 fs_report.cc:383] FS layout report
05:26:29.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:26:29.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-0/wal
05:26:29.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-0/wal
05:26:29.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-0/data/data
05:26:29.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:26:29.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:26:29.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:26:29.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:26:29.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:26:29.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:26:29.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:26:29.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:26:29.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:26:29.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:26:29.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:26:29.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:29.714381 654 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:26:29.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:29.716181 654 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:26:29.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:29.716746 654 kserver.cc:163] Server-wide thread pool size limit: 3276
05:26:29.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:29.720252 654 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
05:26:29.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:29.725564 654 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
05:26:29.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:29.725809 654 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
05:26:29.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:29.726111 654 ts_tablet_manager.cc:610] Registered 0 tablets
05:26:29.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:29.726310 654 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
05:26:29.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:29.930441 654 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.129:37857
05:26:29.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:29.930609 810 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.129:37857 every 8 connection(s)
05:26:29.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:29.933236 654 server_base.cc:1166] Dumped server information to /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-0/data/info.pb
05:26:29.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:29.935767 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-task_eGFIx/build/tsan/bin/kudu as pid 654
05:26:29.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:29.936874 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-0/wal/instance
05:26:29.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:29.967495 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-task_eGFIx/build/tsan/bin/kudu
05:26:29.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_eGFIx/build/tsan/bin/kudu
05:26:29.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-1/wal
05:26:29.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-1/data
05:26:29.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:26:29.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:26:29.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:26:29.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:26:29.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:26:29.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:26:29.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-1/logs
05:26:29.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-1/data/info.pb
05:26:29.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:26:29.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:26:29.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:26:29.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:26:29.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:26:29.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:26:29.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:26:29.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:26:29.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
05:26:29.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:26:29.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.130:0
05:26:29.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.130
05:26:29.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.130
05:26:29.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:26:29.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:42033,127.0.93.189:35555,127.0.93.188:41061
05:26:29.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:46351
05:26:29.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:26:29.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:26:29.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
05:26:30.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:30.007128 811 heartbeater.cc:347] Connected to a master server at 127.0.93.188:41061
05:26:30.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:30.007668 811 heartbeater.cc:464] Registering TS with master...
05:26:30.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:30.010217 811 heartbeater.cc:511] Master 127.0.93.188:41061 requested a full tablet report, sending...
05:26:30.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:30.012176 812 heartbeater.cc:347] Connected to a master server at 127.0.93.189:35555
05:26:30.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:30.012526 812 heartbeater.cc:464] Registering TS with master...
05:26:30.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:30.013334 812 heartbeater.cc:511] Master 127.0.93.189:35555 requested a full tablet report, sending...
05:26:30.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:30.015951 572 ts_manager.cc:194] Registered new tserver with Master: 07491faa591e450aaa7de53fae364b9e (127.0.93.129:37857)
05:26:30.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:30.019528 494 ts_manager.cc:194] Registered new tserver with Master: 07491faa591e450aaa7de53fae364b9e (127.0.93.129:37857)
05:26:30.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:30.023483 815 heartbeater.cc:347] Connected to a master server at 127.0.93.190:42033
05:26:30.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:30.023833 815 heartbeater.cc:464] Registering TS with master...
05:26:30.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:30.024619 815 heartbeater.cc:511] Master 127.0.93.190:42033 requested a full tablet report, sending...
05:26:30.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:30.025718 572 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.93.129:51725
05:26:30.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:30.027734 419 ts_manager.cc:194] Registered new tserver with Master: 07491faa591e450aaa7de53fae364b9e (127.0.93.129:37857)
05:26:30.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:30.334551 821 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:26:30.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:30.334996 821 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:26:30.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:30.335424 821 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:26:30.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:30.366242 821 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:26:30.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:30.367026 821 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.93.130
05:26:30.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:30.400324 821 tablet_server_runner.cc:78] Tablet server non-default flags:
05:26:30.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:26:30.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:46351
05:26:30.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:26:30.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:26:30.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-1/data
05:26:30.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-1/wal
05:26:30.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:26:30.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:26:30.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.130:0
05:26:30.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:26:30.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:26:30.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:26:30.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-1/data/info.pb
05:26:30.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.130
05:26:30.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:26:30.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:42033,127.0.93.189:35555,127.0.93.188:41061
05:26:30.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:26:30.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:26:30.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:26:30.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:26:30.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:26:30.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.130
05:26:30.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-1/logs
05:26:30.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:26:30.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:26:30.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:26:30.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
05:26:30.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:26:30.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:26:30.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:26:30.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 04:01:01 UTC on ccycloud.quasar-phqekz.root.comops.site
05:26:30.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679192
05:26:30.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:26:30.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:30.401525 821 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:26:30.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:30.403010 821 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:26:30.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:30.421347 829 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:26:31.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:31.032406 811 heartbeater.cc:503] Master 127.0.93.188:41061 was elected leader, sending a full tablet report...
05:26:31.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:30.421350 828 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:26:31.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:30.421384 831 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:26:31.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:31.694484 830 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1271 milliseconds
05:26:31.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:31.694594 821 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
05:26:31.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:31.695859 821 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:26:31.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:31.699379 821 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:26:31.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:31.700826 821 hybrid_clock.cc:648] HybridClock initialized: now 1790659591700758 us; error 75 us; skew 500 ppm
05:26:31.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:31.701586 821 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:26:31.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:31.707965 821 webserver.cc:460] Webserver started at http://127.0.93.130:39591/ using document root <none> and password file <none>
05:26:31.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:31.708796 821 fs_manager.cc:359] Metadata directory not provided
05:26:31.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:31.709055 821 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:26:31.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:31.709455 821 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:26:31.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:31.714083 821 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-1/data/instance:
05:26:31.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ae97cc98eb1d4b29b6d85ff0ce53ea42"
05:26:31.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:26:31 on dist-test-slave-gvjk"
05:26:31.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:31.715099 821 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-1/wal/instance:
05:26:31.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ae97cc98eb1d4b29b6d85ff0ce53ea42"
05:26:31.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:26:31 on dist-test-slave-gvjk"
05:26:31.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:31.723235 821 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.010s sys 0.000s
05:26:31.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:31.728469 838 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:26:31.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:31.729535 821 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
05:26:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:31.729874 821 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-1/data,/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-1/wal
05:26:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ae97cc98eb1d4b29b6d85ff0ce53ea42"
05:26:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:26:31 on dist-test-slave-gvjk"
05:26:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:31.730206 821 fs_report.cc:383] FS layout report
05:26:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:26:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-1/wal
05:26:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-1/wal
05:26:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-1/data/data
05:26:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:26:31.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:26:31.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:26:31.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:26:31.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:26:31.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:26:31.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:26:31.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:26:31.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:26:31.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:26:31.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:26:31.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:31.765898 821 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:26:31.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:31.767321 821 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:26:31.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:31.767841 821 kserver.cc:163] Server-wide thread pool size limit: 3276
05:26:31.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:31.770799 821 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
05:26:31.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:31.774663 821 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
05:26:31.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:31.774884 821 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
05:26:31.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:31.775213 821 ts_tablet_manager.cc:610] Registered 0 tablets
05:26:31.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:31.775449 821 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
05:26:31.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:31.920187 821 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.130:39069
05:26:31.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:31.920277 951 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.130:39069 every 8 connection(s)
05:26:31.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:31.922693 821 server_base.cc:1166] Dumped server information to /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-1/data/info.pb
05:26:31.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:31.923419 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-task_eGFIx/build/tsan/bin/kudu as pid 821
05:26:31.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:31.923758 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-1/wal/instance
05:26:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:31.945654 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-task_eGFIx/build/tsan/bin/kudu
05:26:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_eGFIx/build/tsan/bin/kudu
05:26:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-2/wal
05:26:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-2/data
05:26:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:26:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:26:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:26:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:26:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:26:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:26:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-2/logs
05:26:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-2/data/info.pb
05:26:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:26:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:26:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:26:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:26:31.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:26:31.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:26:31.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:26:31.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:26:31.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
05:26:31.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:26:31.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.131:0
05:26:31.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.131
05:26:31.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.131
05:26:31.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:26:31.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:42033,127.0.93.189:35555,127.0.93.188:41061
05:26:31.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:46351
05:26:31.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:26:31.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:26:31.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
05:26:31.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:31.992794 954 heartbeater.cc:347] Connected to a master server at 127.0.93.190:42033
05:26:31.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:31.993194 952 heartbeater.cc:347] Connected to a master server at 127.0.93.188:41061
05:26:31.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:31.993289 954 heartbeater.cc:464] Registering TS with master...
05:26:31.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:31.993965 952 heartbeater.cc:464] Registering TS with master...
05:26:31.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:31.994872 952 heartbeater.cc:511] Master 127.0.93.188:41061 requested a full tablet report, sending...
05:26:31.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:31.995020 954 heartbeater.cc:511] Master 127.0.93.190:42033 requested a full tablet report, sending...
05:26:31.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:31.997411 953 heartbeater.cc:347] Connected to a master server at 127.0.93.189:35555
05:26:31.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:31.997864 953 heartbeater.cc:464] Registering TS with master...
05:26:31.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:31.998621 419 ts_manager.cc:194] Registered new tserver with Master: ae97cc98eb1d4b29b6d85ff0ce53ea42 (127.0.93.130:39069)
05:26:31.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:31.998951 953 heartbeater.cc:511] Master 127.0.93.189:35555 requested a full tablet report, sending...
05:26:32.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:32.001888 494 ts_manager.cc:194] Registered new tserver with Master: ae97cc98eb1d4b29b6d85ff0ce53ea42 (127.0.93.130:39069)
05:26:32.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:32.003530 572 ts_manager.cc:194] Registered new tserver with Master: ae97cc98eb1d4b29b6d85ff0ce53ea42 (127.0.93.130:39069)
05:26:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:32.005113 572 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.93.130:55041
05:26:32.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:32.291474 962 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:26:32.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:32.292474 962 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:26:32.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:32.293074 962 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:26:32.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:32.323742 962 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:26:32.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:32.324736 962 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.93.131
05:26:32.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:32.358496 962 tablet_server_runner.cc:78] Tablet server non-default flags:
05:26:32.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:26:32.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:46351
05:26:32.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:26:32.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:26:32.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-2/data
05:26:32.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-2/wal
05:26:32.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:26:32.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:26:32.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.131:0
05:26:32.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:26:32.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:26:32.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:26:32.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-2/data/info.pb
05:26:32.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.131
05:26:32.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:26:32.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:42033,127.0.93.189:35555,127.0.93.188:41061
05:26:32.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:26:32.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:26:32.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:26:32.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:26:32.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:26:32.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.131
05:26:32.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-2/logs
05:26:32.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:26:32.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:26:32.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:26:32.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
05:26:32.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:26:32.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:26:32.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:26:32.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 04:01:01 UTC on ccycloud.quasar-phqekz.root.comops.site
05:26:32.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679192
05:26:32.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:26:32.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:32.360015 962 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:26:32.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:32.361871 962 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:26:32.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:32.382429 969 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:26:33.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:33.008091 952 heartbeater.cc:503] Master 127.0.93.188:41061 was elected leader, sending a full tablet report...
05:26:33.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:32.383378 968 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:26:33.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:32.384296 962 server_base.cc:1034] running on GCE node
05:26:33.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:32.382454 971 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:26:33.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:33.623427 962 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:26:33.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:33.625946 962 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:26:33.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:33.627441 962 hybrid_clock.cc:648] HybridClock initialized: now 1790659593627396 us; error 38 us; skew 500 ppm
05:26:33.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:33.628592 962 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:26:33.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:33.635332 962 webserver.cc:460] Webserver started at http://127.0.93.131:36103/ using document root <none> and password file <none>
05:26:33.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:33.636301 962 fs_manager.cc:359] Metadata directory not provided
05:26:33.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:33.636494 962 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:26:33.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:33.636886 962 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:26:33.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:33.641719 962 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-2/data/instance:
05:26:33.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "adbaf10fc60b4cf7884242acfb9050d6"
05:26:33.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:26:33 on dist-test-slave-gvjk"
05:26:33.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:33.642784 962 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-2/wal/instance:
05:26:33.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "adbaf10fc60b4cf7884242acfb9050d6"
05:26:33.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:26:33 on dist-test-slave-gvjk"
05:26:33.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:33.649852 962 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.005s sys 0.004s
05:26:33.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:33.655263 979 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:26:33.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:33.656391 962 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
05:26:33.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:33.656726 962 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-2/data,/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-2/wal
05:26:33.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "adbaf10fc60b4cf7884242acfb9050d6"
05:26:33.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:26:33 on dist-test-slave-gvjk"
05:26:33.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:33.657056 962 fs_report.cc:383] FS layout report
05:26:33.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:26:33.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-2/wal
05:26:33.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-2/wal
05:26:33.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-2/data/data
05:26:33.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:26:33.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:26:33.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:26:33.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:26:33.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:26:33.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:26:33.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:26:33.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:26:33.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:26:33.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:26:33.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:26:33.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:33.709908 962 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:26:33.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:33.711396 962 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:26:33.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:33.711928 962 kserver.cc:163] Server-wide thread pool size limit: 3276
05:26:33.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:33.714614 962 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
05:26:33.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:33.718869 962 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
05:26:33.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:33.719087 962 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
05:26:33.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:33.719429 962 ts_tablet_manager.cc:610] Registered 0 tablets
05:26:33.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:33.719611 962 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
05:26:33.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:33.860689 962 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.131:44329
05:26:33.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:33.860780 1091 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.131:44329 every 8 connection(s)
05:26:33.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:33.863067 962 server_base.cc:1166] Dumped server information to /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-2/data/info.pb
05:26:33.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:33.870250 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-task_eGFIx/build/tsan/bin/kudu as pid 962
05:26:33.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:33.870726 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster2202097516167667676/ts-2/wal/instance
05:26:33.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:33.912705 1092 heartbeater.cc:347] Connected to a master server at 127.0.93.188:41061
05:26:33.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:33.912796 1093 heartbeater.cc:347] Connected to a master server at 127.0.93.189:35555
05:26:33.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:33.912926 1094 heartbeater.cc:347] Connected to a master server at 127.0.93.190:42033
05:26:33.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:33.913218 1092 heartbeater.cc:464] Registering TS with master...
05:26:33.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:33.913255 1093 heartbeater.cc:464] Registering TS with master...
05:26:33.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:33.913286 1094 heartbeater.cc:464] Registering TS with master...
05:26:33.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:33.914547 1094 heartbeater.cc:511] Master 127.0.93.190:42033 requested a full tablet report, sending...
05:26:33.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:33.914706 1092 heartbeater.cc:511] Master 127.0.93.188:41061 requested a full tablet report, sending...
05:26:33.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:33.915287 1093 heartbeater.cc:511] Master 127.0.93.189:35555 requested a full tablet report, sending...
05:26:33.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:33.917539 572 ts_manager.cc:194] Registered new tserver with Master: adbaf10fc60b4cf7884242acfb9050d6 (127.0.93.131:44329)
05:26:33.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:33.917940 494 ts_manager.cc:194] Registered new tserver with Master: adbaf10fc60b4cf7884242acfb9050d6 (127.0.93.131:44329)
05:26:33.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:33.919771 418 ts_manager.cc:194] Registered new tserver with Master: adbaf10fc60b4cf7884242acfb9050d6 (127.0.93.131:44329)
05:26:33.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:33.919894 572 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.93.131:38107
05:26:33.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:33.928308 374 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
05:26:33.939 [DEBUG - main] (MiniKuduCluster.java:195) Response:
05:26:33.941 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
05:26:33.951 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.0.93.190"
port: 42033
}
bound_http_address {
host: "127.0.93.190"
port: 38567
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.0.93.189"
port: 35555
}
bound_http_address {
host: "127.0.93.189"
port: 43871
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.0.93.188"
port: 41061
}
bound_http_address {
host: "127.0.93.188"
port: 34413
}
}
}
05:26:33.955 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
05:26:33.958 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.0.93.129"
port: 37857
}
bound_http_address {
host: "127.0.93.129"
port: 38715
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.0.93.130"
port: 39069
}
bound_http_address {
host: "127.0.93.130"
port: 39591
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.0.93.131"
port: 44329
}
bound_http_address {
host: "127.0.93.131"
port: 36103
}
}
}
05:26:33.958 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
05:26:34.132 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
05:26:34.254 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task_eGFIx/test-tmp
05:26:34.255 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
05:26:34.256 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task_eGFIx/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
05:26:34.259 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
05:26:34.591 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:26:34.591 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
05:26:34.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:34.595080 1103 mini_oidc.cc:152] Starting JWKS server
05:26:34.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:34.598917 1103 webserver.cc:460] Webserver started at http://127.0.0.1:43121/ using document root <none> and password file <none>
05:26:34.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:34.599354 1103 mini_oidc.cc:189] Starting OIDC Discovery server
05:26:34.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:34.601903 1103 webserver.cc:460] Webserver started at http://127.0.0.1:45989/ using document root <none> and password file <none>
05:26:34.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
05:26:34.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/krb5kdc/principal' for realm 'KRBTEST.COM',
05:26:34.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
05:26:34.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:26:34 dist-test-slave-gvjk krb5kdc[1115](info): setting up network...
05:26:34.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
05:26:34.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:26:34 dist-test-slave-gvjk krb5kdc[1115](info): set up 2 sockets
05:26:34.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:26:34 dist-test-slave-gvjk krb5kdc[1115](info): commencing operation
05:26:34.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
05:26:34.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:34.926041 1092 heartbeater.cc:503] Master 127.0.93.188:41061 was elected leader, sending a full tablet report...
05:26:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:35.764250 1103 mini_kdc.cc:121] Time spent starting KDC: real 1.135s user 0.000s sys 0.022s
05:26:35.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
05:26:35.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
05:26:35.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
05:26:35.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
05:26:35.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
05:26:35.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
05:26:35.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
05:26:35.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
05:26:35.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
05:26:35.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
05:26:35.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/krb5kdc/test-user.keytab.
05:26:35.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/krb5kdc/test-user.keytab.
05:26:35.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:26:35 dist-test-slave-gvjk krb5kdc[1115](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790659595, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:26:35.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
05:26:35.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T05:26:35Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
05:26:35.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T05:26:35Z Disabled control of system clock
05:26:35.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.19.254@KRBTEST.COM; defaulting to no policy
05:26:35.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:26:35.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.19.254@KRBTEST.COM" created.
05:26:36.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:26:36.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/krb5kdc/kudu_127.1.19.254.keytab.
05:26:36.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/krb5kdc/kudu_127.1.19.254.keytab.
05:26:36.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:36.076597 1103 mini_kdc.cc:262] Time spent creating service keytab for kudu/127.1.19.254: real 0.150s user 0.004s sys 0.008s
05:26:36.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:36.078529 1103 external_mini_cluster.cc:1326] Running /tmp/dist-test-task_eGFIx/build/tsan/bin/kudu
05:26:36.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_eGFIx/build/tsan/bin/kudu
05:26:36.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-0/wal
05:26:36.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-0/data
05:26:36.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:26:36.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:26:36.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:26:36.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:26:36.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:26:36.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:26:36.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-0/logs
05:26:36.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-0/data/info.pb
05:26:36.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:26:36.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:26:36.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:26:36.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:26:36.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:26:36.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:26:36.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:26:36.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:26:36.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:26:36.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:26:36.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:26:36.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:26:36.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.254:45545
05:26:36.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.254
05:26:36.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:26:36.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/krb5kdc/kudu_127.1.19.254.keytab
05:26:36.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.254
05:26:36.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:26:36.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:26:36.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:26:36.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.212:36119
05:26:36.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:26:36.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:26:36.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:26:36.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:26:36.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.254:45545,127.1.19.253:39909,127.1.19.252:34617
05:26:36.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:26:36.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43121/jwks.json
05:26:36.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:26:36.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:36.404489 1128 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:26:36.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:36.405560 1128 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:26:36.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:36.406255 1128 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:26:36.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:36.437072 1128 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:26:36.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:36.437345 1128 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:26:36.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:36.437716 1128 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:26:36.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:36.438036 1128 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:26:36.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:36.438344 1128 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:26:36.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:36.438674 1128 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:26:36.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:36.438933 1128 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43121/jwks.json
05:26:36.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:36.479120 1128 master_runner.cc:386] Master server non-default flags:
05:26:36.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:26:36.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.212:36119
05:26:36.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:26:36.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:26:36.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-0/data
05:26:36.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-0/wal
05:26:36.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:26:36.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:26:36.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.254:45545,127.1.19.253:39909,127.1.19.252:34617
05:26:36.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/krb5kdc/kudu_127.1.19.254.keytab
05:26:36.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.254
05:26:36.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:26:36.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:26:36.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:26:36.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.254:45545
05:26:36.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:26:36.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:26:36.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:26:36.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43121/jwks.json
05:26:36.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:26:36.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:26:36.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:26:36.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:26:36.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:26:36.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-0/data/info.pb
05:26:36.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.254
05:26:36.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:26:36.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:26:36.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:26:36.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:26:36.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:26:36.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:26:36.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-0/logs
05:26:36.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:26:36.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:26:36.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:26:36.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:26:36.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:26:36.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:26:36.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:26:36.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 04:01:01 UTC on ccycloud.quasar-phqekz.root.comops.site
05:26:36.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679192
05:26:36.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:26:36.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:36.480280 1128 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:26:36.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:36.481521 1128 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:26:36.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:36.504501 1135 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:26:37.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:36.504958 1138 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:26:37.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:36.505818 1136 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:26:37.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:36.507673 1128 server_base.cc:1034] running on GCE node
05:26:37.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:37.744508 1128 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:26:37.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:37.747872 1128 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:26:37.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:37.749253 1128 hybrid_clock.cc:648] HybridClock initialized: now 1790659597749195 us; error 90 us; skew 500 ppm
05:26:37.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:37.750239 1128 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:26:37.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:26:37 dist-test-slave-gvjk krb5kdc[1115](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790659597, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:26:37.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:37.756475 1128 init.cc:377] Logged in from keytab as kudu/127.1.19.254@KRBTEST.COM (short username kudu)
05:26:37.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:37.761754 1128 webserver.cc:460] Webserver started at http://127.1.19.254:40567/ using document root <none> and password file <none>
05:26:37.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:37.762843 1128 fs_manager.cc:359] Metadata directory not provided
05:26:37.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:37.763089 1128 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:26:37.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:37.763659 1128 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:26:37.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:37.769289 1128 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-0/data/instance:
05:26:37.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "82218c588ded4617b3da9bc194cc51c9"
05:26:37.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:26:37 on dist-test-slave-gvjk"
05:26:37.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:37.770684 1128 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-0/wal/instance:
05:26:37.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "82218c588ded4617b3da9bc194cc51c9"
05:26:37.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:26:37 on dist-test-slave-gvjk"
05:26:37.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:37.778822 1128 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
05:26:37.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:37.784821 1146 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:26:37.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:37.786031 1128 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
05:26:37.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:37.786413 1128 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-0/data,/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-0/wal
05:26:37.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "82218c588ded4617b3da9bc194cc51c9"
05:26:37.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:26:37 on dist-test-slave-gvjk"
05:26:37.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:37.786891 1128 fs_report.cc:383] FS layout report
05:26:37.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:26:37.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-0/wal
05:26:37.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-0/wal
05:26:37.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-0/data/data
05:26:37.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:26:37.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:26:37.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:26:37.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:26:37.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:26:37.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:26:37.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:26:37.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:26:37.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:26:37.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:26:37.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:26:37.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:37.850687 1128 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:26:37.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:37.856570 1128 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:26:37.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:37.856964 1128 kserver.cc:163] Server-wide thread pool size limit: 3276
05:26:37.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:37.941387 1128 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.254:45545
05:26:37.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:37.941476 1197 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.254:45545 every 8 connection(s)
05:26:37.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:37.945616 1128 server_base.cc:1166] Dumped server information to /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-0/data/info.pb
05:26:37.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:37.950285 1103 external_mini_cluster.cc:1388] Started /tmp/dist-test-task_eGFIx/build/tsan/bin/kudu as pid 1128
05:26:37.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:37.950645 1103 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-0/wal/instance
05:26:37.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:37.950969 1198 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:26:37.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:37.966740 1198 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.254" port: 45545 } has no permanent_uuid. Determining permanent_uuid...
05:26:37.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.19.253@KRBTEST.COM; defaulting to no policy
05:26:37.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:26:37.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.19.253@KRBTEST.COM" created.
05:26:37.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:26:37 dist-test-slave-gvjk krb5kdc[1115](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790659597, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.254@KRBTEST.COM for kudu/127.1.19.254@KRBTEST.COM
05:26:37.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:26:37.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/krb5kdc/kudu_127.1.19.253.keytab.
05:26:37.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/krb5kdc/kudu_127.1.19.253.keytab.
05:26:38.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:38.000957 1103 external_mini_cluster.cc:1326] Running /tmp/dist-test-task_eGFIx/build/tsan/bin/kudu
05:26:38.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_eGFIx/build/tsan/bin/kudu
05:26:38.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-1/wal
05:26:38.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-1/data
05:26:38.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:26:38.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:26:38.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:26:38.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:26:38.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:26:38.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:26:38.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-1/logs
05:26:38.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-1/data/info.pb
05:26:38.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:26:38.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:26:38.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:26:38.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:26:38.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:26:38.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:26:38.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:26:38.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:26:38.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:26:38.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:26:38.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:26:38.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:26:38.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.253:39909
05:26:38.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.253
05:26:38.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:26:38.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/krb5kdc/kudu_127.1.19.253.keytab
05:26:38.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.253
05:26:38.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:26:38.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:26:38.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:26:38.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.212:36119
05:26:38.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:26:38.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:26:38.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:26:38.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:26:38.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.254:45545,127.1.19.253:39909,127.1.19.252:34617
05:26:38.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:26:38.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43121/jwks.json
05:26:38.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:26:38.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:38.016181 1198 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.253" port: 39909 } has no permanent_uuid. Determining permanent_uuid...
05:26:38.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:38.019937 1150 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.253:39909: connect: Connection refused (error 111)
05:26:38.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:38.028106 1198 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.253:39909: Network error: Client connection negotiation failed: client connection to 127.1.19.253:39909: connect: Connection refused (error 111)
05:26:38.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:38.077699 1198 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.253" port: 39909 } attempt: 1
05:26:38.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:38.081430 1198 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.253:39909: Network error: Client connection negotiation failed: client connection to 127.1.19.253:39909: connect: Connection refused (error 111)
05:26:38.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:38.149863 1198 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.253" port: 39909 } attempt: 2
05:26:38.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:38.153390 1198 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.253:39909: Network error: Client connection negotiation failed: client connection to 127.1.19.253:39909: connect: Connection refused (error 111)
05:26:38.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:38.244850 1198 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.253" port: 39909 } attempt: 3
05:26:38.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:38.249336 1198 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.253:39909: Network error: Client connection negotiation failed: client connection to 127.1.19.253:39909: connect: Connection refused (error 111)
05:26:38.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:38.331019 1204 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:26:38.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:38.331545 1204 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:26:38.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:38.332376 1204 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:26:38.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:38.365913 1204 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:26:38.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:38.366142 1204 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:26:38.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:38.366405 1204 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:26:38.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:38.366597 1204 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:26:38.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:38.366781 1204 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:26:38.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:38.366967 1204 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:26:38.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:38.367105 1204 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43121/jwks.json
05:26:38.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:38.392877 1198 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.253" port: 39909 } attempt: 4
05:26:38.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:38.396586 1198 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.253:39909: Network error: Client connection negotiation failed: client connection to 127.1.19.253:39909: connect: Connection refused (error 111)
05:26:38.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:38.404860 1204 master_runner.cc:386] Master server non-default flags:
05:26:38.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:26:38.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.212:36119
05:26:38.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:26:38.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:26:38.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-1/data
05:26:38.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-1/wal
05:26:38.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:26:38.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:26:38.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.254:45545,127.1.19.253:39909,127.1.19.252:34617
05:26:38.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/krb5kdc/kudu_127.1.19.253.keytab
05:26:38.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.253
05:26:38.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:26:38.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:26:38.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:26:38.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.253:39909
05:26:38.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:26:38.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:26:38.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:26:38.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43121/jwks.json
05:26:38.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:26:38.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:26:38.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:26:38.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:26:38.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:26:38.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-1/data/info.pb
05:26:38.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.253
05:26:38.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:26:38.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:26:38.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:26:38.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:26:38.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:26:38.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:26:38.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-1/logs
05:26:38.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:26:38.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:26:38.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:26:38.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:26:38.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:26:38.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:26:38.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:26:38.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 04:01:01 UTC on ccycloud.quasar-phqekz.root.comops.site
05:26:38.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679192
05:26:38.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:26:38.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:38.405920 1204 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:26:38.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:38.406977 1204 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:26:38.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:38.426440 1213 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:26:38.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:38.695978 1198 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.253" port: 39909 } attempt: 5
05:26:38.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:38.699510 1198 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.253:39909: Network error: Client connection negotiation failed: client connection to 127.1.19.253:39909: connect: Connection refused (error 111)
05:26:39.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:39.247108 1198 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.253" port: 39909 } attempt: 6
05:26:39.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:39.255153 1198 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.253:39909: Network error: Client connection negotiation failed: client connection to 127.1.19.253:39909: connect: Connection refused (error 111)
05:26:39.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:38.426524 1212 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:26:39.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:38.426440 1215 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:26:39.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:38.428157 1204 server_base.cc:1034] running on GCE node
05:26:39.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:39.684491 1204 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:26:39.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:39.687940 1204 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:26:39.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:39.689425 1204 hybrid_clock.cc:648] HybridClock initialized: now 1790659599689386 us; error 66 us; skew 500 ppm
05:26:39.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:39.690295 1204 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:26:39.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:26:39 dist-test-slave-gvjk krb5kdc[1115](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790659599, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:26:39.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:39.701126 1204 init.cc:377] Logged in from keytab as kudu/127.1.19.253@KRBTEST.COM (short username kudu)
05:26:39.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:39.706947 1204 webserver.cc:460] Webserver started at http://127.1.19.253:42821/ using document root <none> and password file <none>
05:26:39.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:39.707954 1204 fs_manager.cc:359] Metadata directory not provided
05:26:39.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:39.708221 1204 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:26:39.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:39.708812 1204 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:26:39.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:39.714012 1204 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-1/data/instance:
05:26:39.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2fbfabd1811349149652cf260cc23e13"
05:26:39.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:26:39 on dist-test-slave-gvjk"
05:26:39.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:39.715121 1204 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-1/wal/instance:
05:26:39.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2fbfabd1811349149652cf260cc23e13"
05:26:39.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:26:39 on dist-test-slave-gvjk"
05:26:39.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:39.723094 1204 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.005s
05:26:39.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:39.728794 1226 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:26:39.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:39.729978 1204 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
05:26:39.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:39.730252 1204 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-1/data,/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-1/wal
05:26:39.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2fbfabd1811349149652cf260cc23e13"
05:26:39.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:26:39 on dist-test-slave-gvjk"
05:26:39.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:39.730561 1204 fs_report.cc:383] FS layout report
05:26:39.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:26:39.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-1/wal
05:26:39.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-1/wal
05:26:39.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-1/data/data
05:26:39.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:26:39.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:26:39.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:26:39.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:26:39.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:26:39.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:26:39.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:26:39.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:26:39.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:26:39.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:26:39.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:26:39.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:39.780423 1204 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:26:39.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:39.786368 1204 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:26:39.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:39.786796 1204 kserver.cc:163] Server-wide thread pool size limit: 3276
05:26:39.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:39.867208 1204 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.253:39909
05:26:39.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:39.867370 1277 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.253:39909 every 8 connection(s)
05:26:39.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:39.869987 1204 server_base.cc:1166] Dumped server information to /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-1/data/info.pb
05:26:39.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:39.875715 1278 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:26:39.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:39.879397 1103 external_mini_cluster.cc:1388] Started /tmp/dist-test-task_eGFIx/build/tsan/bin/kudu as pid 1204
05:26:39.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:39.879844 1103 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-1/wal/instance
05:26:39.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:39.887524 1278 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.254" port: 45545 } has no permanent_uuid. Determining permanent_uuid...
05:26:39.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.19.252@KRBTEST.COM; defaulting to no policy
05:26:39.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:26:39.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.19.252@KRBTEST.COM" created.
05:26:39.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:26:39 dist-test-slave-gvjk krb5kdc[1115](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790659599, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.253@KRBTEST.COM for kudu/127.1.19.254@KRBTEST.COM
05:26:39.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:26:39.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/krb5kdc/kudu_127.1.19.252.keytab.
05:26:39.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/krb5kdc/kudu_127.1.19.252.keytab.
05:26:39.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:39.930091 1103 external_mini_cluster.cc:1326] Running /tmp/dist-test-task_eGFIx/build/tsan/bin/kudu
05:26:39.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_eGFIx/build/tsan/bin/kudu
05:26:39.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-2/wal
05:26:39.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-2/data
05:26:39.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:26:39.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:26:39.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:26:39.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:26:39.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:26:39.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:26:39.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-2/logs
05:26:39.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-2/data/info.pb
05:26:39.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:26:39.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:26:39.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:26:39.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:26:39.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:26:39.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:26:39.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:26:39.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:26:39.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:26:39.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:26:39.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:26:39.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:26:39.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.252:34617
05:26:39.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.252
05:26:39.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:26:39.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/krb5kdc/kudu_127.1.19.252.keytab
05:26:39.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.252
05:26:39.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:26:39.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:26:39.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:26:39.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.212:36119
05:26:39.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:26:39.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:26:39.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:26:39.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:26:39.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.254:45545,127.1.19.253:39909,127.1.19.252:34617
05:26:39.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:26:39.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43121/jwks.json
05:26:39.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:26:39.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:39.932574 1278 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.253" port: 39909 } has no permanent_uuid. Determining permanent_uuid...
05:26:39.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:26:39 dist-test-slave-gvjk krb5kdc[1115](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790659599, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.253@KRBTEST.COM for kudu/127.1.19.253@KRBTEST.COM
05:26:39.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:39.969079 1278 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.252" port: 34617 } has no permanent_uuid. Determining permanent_uuid...
05:26:39.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:39.971426 1229 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.252:34617: connect: Connection refused (error 111)
05:26:39.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:39.975965 1278 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.252:34617: Network error: Client connection negotiation failed: client connection to 127.1.19.252:34617: connect: Connection refused (error 111)
05:26:40.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:40.025403 1278 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.252" port: 34617 } attempt: 1
05:26:40.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:40.029426 1278 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.252:34617: Network error: Client connection negotiation failed: client connection to 127.1.19.252:34617: connect: Connection refused (error 111)
05:26:40.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:40.097833 1278 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.252" port: 34617 } attempt: 2
05:26:40.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:40.101527 1278 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.252:34617: Network error: Client connection negotiation failed: client connection to 127.1.19.252:34617: connect: Connection refused (error 111)
05:26:40.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:40.192973 1278 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.252" port: 34617 } attempt: 3
05:26:40.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:40.196782 1278 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.252:34617: Network error: Client connection negotiation failed: client connection to 127.1.19.252:34617: connect: Connection refused (error 111)
05:26:40.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:40.261824 1285 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:26:40.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:40.262363 1285 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:26:40.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:40.262931 1285 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:26:40.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:40.294633 1285 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:26:40.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:40.294857 1285 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:26:40.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:40.295109 1285 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:26:40.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:40.295320 1285 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:26:40.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:40.295506 1285 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:26:40.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:40.295688 1285 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:26:40.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:40.295902 1285 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43121/jwks.json
05:26:40.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:40.315572 1198 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.253" port: 39909 } attempt: 7
05:26:40.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:26:40 dist-test-slave-gvjk krb5kdc[1115](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790659597, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.254@KRBTEST.COM for kudu/127.1.19.253@KRBTEST.COM
05:26:40.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:40.340152 1278 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.252" port: 34617 } attempt: 4
05:26:40.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:40.340852 1285 master_runner.cc:386] Master server non-default flags:
05:26:40.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:26:40.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.212:36119
05:26:40.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:26:40.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:26:40.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-2/data
05:26:40.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-2/wal
05:26:40.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:26:40.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:26:40.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.254:45545,127.1.19.253:39909,127.1.19.252:34617
05:26:40.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/krb5kdc/kudu_127.1.19.252.keytab
05:26:40.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.252
05:26:40.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:26:40.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:26:40.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:26:40.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.252:34617
05:26:40.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:26:40.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:26:40.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:26:40.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43121/jwks.json
05:26:40.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:26:40.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:26:40.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:26:40.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:26:40.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:26:40.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-2/data/info.pb
05:26:40.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.252
05:26:40.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:26:40.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:26:40.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:26:40.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:26:40.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:26:40.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:26:40.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-2/logs
05:26:40.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:26:40.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:26:40.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:26:40.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:26:40.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:26:40.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:26:40.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:26:40.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 04:01:01 UTC on ccycloud.quasar-phqekz.root.comops.site
05:26:40.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679192
05:26:40.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:26:40.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:40.342363 1285 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:26:40.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:40.343601 1285 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:26:40.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:40.344133 1278 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.252:34617: Network error: Client connection negotiation failed: client connection to 127.1.19.252:34617: connect: Connection refused (error 111)
05:26:40.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:40.346880 1198 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.252" port: 34617 } has no permanent_uuid. Determining permanent_uuid...
05:26:40.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:40.354529 1198 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.252:34617: Network error: Client connection negotiation failed: client connection to 127.1.19.252:34617: connect: Connection refused (error 111)
05:26:40.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:40.362996 1297 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:26:40.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:40.413050 1198 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.252" port: 34617 } attempt: 1
05:26:40.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:40.416539 1198 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.252:34617: Network error: Client connection negotiation failed: client connection to 127.1.19.252:34617: connect: Connection refused (error 111)
05:26:40.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:40.498116 1198 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.252" port: 34617 } attempt: 2
05:26:40.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:40.501827 1198 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.252:34617: Network error: Client connection negotiation failed: client connection to 127.1.19.252:34617: connect: Connection refused (error 111)
05:26:40.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:40.587332 1198 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.252" port: 34617 } attempt: 3
05:26:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:40.591658 1198 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.252:34617: Network error: Client connection negotiation failed: client connection to 127.1.19.252:34617: connect: Connection refused (error 111)
05:26:40.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:40.643586 1278 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.252" port: 34617 } attempt: 5
05:26:40.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:40.647396 1278 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.252:34617: Network error: Client connection negotiation failed: client connection to 127.1.19.252:34617: connect: Connection refused (error 111)
05:26:40.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:40.732137 1198 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.252" port: 34617 } attempt: 4
05:26:40.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:40.735704 1198 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.252:34617: Network error: Client connection negotiation failed: client connection to 127.1.19.252:34617: connect: Connection refused (error 111)
05:26:41.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:41.019315 1198 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.252" port: 34617 } attempt: 5
05:26:41.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:41.023263 1198 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.252:34617: Network error: Client connection negotiation failed: client connection to 127.1.19.252:34617: connect: Connection refused (error 111)
05:26:41.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:41.194995 1278 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.252" port: 34617 } attempt: 6
05:26:41.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:41.202663 1278 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.252:34617: Network error: Client connection negotiation failed: client connection to 127.1.19.252:34617: connect: Connection refused (error 111)
05:26:41.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:41.575843 1198 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.252" port: 34617 } attempt: 6
05:26:41.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:41.582747 1198 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.252:34617: Network error: Client connection negotiation failed: client connection to 127.1.19.252:34617: connect: Connection refused (error 111)
05:26:41.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:40.363070 1296 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:26:41.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:40.364037 1299 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:26:41.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:41.640265 1298 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
05:26:41.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:41.640419 1285 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
05:26:41.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:41.644985 1285 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:26:41.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:41.647528 1285 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:26:41.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:41.648993 1285 hybrid_clock.cc:648] HybridClock initialized: now 1790659601648944 us; error 100 us; skew 500 ppm
05:26:41.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:41.649927 1285 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:26:41.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:26:41 dist-test-slave-gvjk krb5kdc[1115](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790659601, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:26:41.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:41.656183 1285 init.cc:377] Logged in from keytab as kudu/127.1.19.252@KRBTEST.COM (short username kudu)
05:26:41.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:41.662009 1285 webserver.cc:460] Webserver started at http://127.1.19.252:41891/ using document root <none> and password file <none>
05:26:41.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:41.662966 1285 fs_manager.cc:359] Metadata directory not provided
05:26:41.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:41.663229 1285 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:26:41.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:41.663713 1285 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:26:41.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:41.668720 1285 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-2/data/instance:
05:26:41.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "238e07129b03453d8f3bb2f71dfd1c9c"
05:26:41.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:26:41 on dist-test-slave-gvjk"
05:26:41.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:41.669992 1285 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-2/wal/instance:
05:26:41.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "238e07129b03453d8f3bb2f71dfd1c9c"
05:26:41.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:26:41 on dist-test-slave-gvjk"
05:26:41.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:41.678058 1285 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.002s sys 0.007s
05:26:41.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:41.685312 1311 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:26:41.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:41.686678 1285 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.003s
05:26:41.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:41.687027 1285 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-2/data,/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-2/wal
05:26:41.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "238e07129b03453d8f3bb2f71dfd1c9c"
05:26:41.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:26:41 on dist-test-slave-gvjk"
05:26:41.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:41.687388 1285 fs_report.cc:383] FS layout report
05:26:41.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:26:41.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-2/wal
05:26:41.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-2/wal
05:26:41.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-2/data/data
05:26:41.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:26:41.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:26:41.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:26:41.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:26:41.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:26:41.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:26:41.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:26:41.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:26:41.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:26:41.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:26:41.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:26:41.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:41.729627 1285 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:26:41.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:41.736450 1285 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:26:41.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:41.736864 1285 kserver.cc:163] Server-wide thread pool size limit: 3276
05:26:41.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:41.824453 1285 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.252:34617
05:26:41.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:41.824518 1363 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.252:34617 every 8 connection(s)
05:26:41.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:41.827068 1285 server_base.cc:1166] Dumped server information to /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-2/data/info.pb
05:26:41.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:41.830783 1103 external_mini_cluster.cc:1388] Started /tmp/dist-test-task_eGFIx/build/tsan/bin/kudu as pid 1285
05:26:41.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:41.831245 1103 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-2/wal/instance
05:26:41.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:41.834745 1364 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:26:41.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:41.848270 1364 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.254" port: 45545 } has no permanent_uuid. Determining permanent_uuid...
05:26:41.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:26:41 dist-test-slave-gvjk krb5kdc[1115](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790659601, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.252@KRBTEST.COM for kudu/127.1.19.254@KRBTEST.COM
05:26:41.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:26:41 dist-test-slave-gvjk krb5kdc[1115](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790659595, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.19.252@KRBTEST.COM
05:26:41.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:41.893353 1364 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.253" port: 39909 } has no permanent_uuid. Determining permanent_uuid...
05:26:41.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:26:41 dist-test-slave-gvjk krb5kdc[1115](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790659601, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.252@KRBTEST.COM for kudu/127.1.19.253@KRBTEST.COM
05:26:41.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:26:41 dist-test-slave-gvjk krb5kdc[1115](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790659595, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.19.253@KRBTEST.COM
05:26:41.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:41.922869 1364 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.252" port: 34617 } has no permanent_uuid. Determining permanent_uuid...
05:26:41.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:26:41 dist-test-slave-gvjk krb5kdc[1115](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790659601, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.252@KRBTEST.COM for kudu/127.1.19.252@KRBTEST.COM
05:26:41.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:26:41 dist-test-slave-gvjk krb5kdc[1115](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790659595, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.19.254@KRBTEST.COM
05:26:41.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:41.958765 1364 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c: Bootstrap starting.
05:26:41.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:41.966151 1364 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c: Neither blocks nor log segments found. Creating new log.
05:26:41.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:41.967533 1103 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
05:26:41.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:41.968420 1364 log.cc:824] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c: Log is configured to *not* fsync() on all Append() calls
05:26:41.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:41.974160 1364 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c: No bootstrap required, opened a new log
05:26:41.978 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:26:41.979 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
05:26:41.991 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
05:26:41.991 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
05:26:41.994 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.1.19.254"
port: 45545
}
bound_http_address {
host: "127.1.19.254"
port: 40567
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.1.19.253"
port: 39909
}
bound_http_address {
host: "127.1.19.253"
port: 42821
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.1.19.252"
port: 34617
}
bound_http_address {
host: "127.1.19.252"
port: 41891
}
}
}
05:26:41.995 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
05:26:41.995 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:26:41.995 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
05:26:41.996 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
05:26:42.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:42.001092 1364 raft_consensus.cc:348] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82218c588ded4617b3da9bc194cc51c9" member_type: VOTER last_known_addr { host: "127.1.19.254" port: 45545 } } peers { permanent_uuid: "2fbfabd1811349149652cf260cc23e13" member_type: VOTER last_known_addr { host: "127.1.19.253" port: 39909 } } peers { permanent_uuid: "238e07129b03453d8f3bb2f71dfd1c9c" member_type: VOTER last_known_addr { host: "127.1.19.252" port: 34617 } }
05:26:42.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:42.001775 1364 raft_consensus.cc:374] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
05:26:42.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:42.002063 1364 raft_consensus.cc:729] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 238e07129b03453d8f3bb2f71dfd1c9c, State: Initialized, Role: FOLLOWER
05:26:42.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:42.002933 1364 consensus_queue.cc:260] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82218c588ded4617b3da9bc194cc51c9" member_type: VOTER last_known_addr { host: "127.1.19.254" port: 45545 } } peers { permanent_uuid: "2fbfabd1811349149652cf260cc23e13" member_type: VOTER last_known_addr { host: "127.1.19.253" port: 39909 } } peers { permanent_uuid: "238e07129b03453d8f3bb2f71dfd1c9c" member_type: VOTER last_known_addr { host: "127.1.19.252" port: 34617 } }
05:26:42.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:42.006181 1374 sys_catalog.cc:455] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82218c588ded4617b3da9bc194cc51c9" member_type: VOTER last_known_addr { host: "127.1.19.254" port: 45545 } } peers { permanent_uuid: "2fbfabd1811349149652cf260cc23e13" member_type: VOTER last_known_addr { host: "127.1.19.253" port: 39909 } } peers { permanent_uuid: "238e07129b03453d8f3bb2f71dfd1c9c" member_type: VOTER last_known_addr { host: "127.1.19.252" port: 34617 } } }
05:26:42.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:42.007187 1374 sys_catalog.cc:458] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [sys.catalog]: This master's current role is: FOLLOWER
05:26:42.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:42.008075 1364 sys_catalog.cc:564] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [sys.catalog]: configured and running, proceeding with master startup.
05:26:42.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:26:42 dist-test-slave-gvjk krb5kdc[1115](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790659602, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:26:42.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
05:26:42.035 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:26:42.038 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/krb5kdc/krb5cc
05:26:42.046 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
05:26:42.047 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/krb5kdc/krb5cc
05:26:42.048 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
05:26:42.054 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
05:26:42.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:42.055327 1386 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:26:42.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:42.055660 1386 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:26:42.057 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
05:26:42.057 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
05:26:42.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:42.063524 1374 raft_consensus.cc:482] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:26:42.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:42.064144 1374 raft_consensus.cc:504] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82218c588ded4617b3da9bc194cc51c9" member_type: VOTER last_known_addr { host: "127.1.19.254" port: 45545 } } peers { permanent_uuid: "2fbfabd1811349149652cf260cc23e13" member_type: VOTER last_known_addr { host: "127.1.19.253" port: 39909 } } peers { permanent_uuid: "238e07129b03453d8f3bb2f71dfd1c9c" member_type: VOTER last_known_addr { host: "127.1.19.252" port: 34617 } }
05:26:42.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:42.070509 1374 leader_election.cc:288] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82218c588ded4617b3da9bc194cc51c9 (127.1.19.254:45545), 2fbfabd1811349149652cf260cc23e13 (127.1.19.253:39909)
05:26:42.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:42.035128 1285 master_runner.cc:186] Error getting master registration for 127.1.19.254:45545: OK, instance_id { permanent_uuid: "82218c588ded4617b3da9bc194cc51c9" instance_seqno: 1790659597915891 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
05:26:42.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:42.083120 1285 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:26:42.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:26:42 dist-test-slave-gvjk krb5kdc[1115](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790659602, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:26:42.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
05:26:42.100 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:26:42.100 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/krb5kdc/krb5cc
05:26:42.102 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
05:26:42.102 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.19.254:45545
05:26:42.109 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
05:26:42.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:42.110258 1103 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task_eGFIx/build/tsan/bin/kudu with pid 1128
05:26:42.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:42.137681 1313 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv got EOF from 127.1.19.254:45545 (error 108)
05:26:42.145 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:26:42.145 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.19.252:34617
05:26:42.145 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
05:26:42.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:42.146509 1103 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task_eGFIx/build/tsan/bin/kudu with pid 1285
05:26:42.170 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:26:42.171 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.19.253:39909
05:26:42.171 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
05:26:42.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:42.171504 1103 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task_eGFIx/build/tsan/bin/kudu with pid 1204
05:26:42.192 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:26:42.192 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.19.254:45545
05:26:42.194 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
05:26:42.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:42.195312 1103 external_mini_cluster.cc:1326] Running /tmp/dist-test-task_eGFIx/build/tsan/bin/kudu
05:26:42.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_eGFIx/build/tsan/bin/kudu
05:26:42.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-0/wal
05:26:42.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-0/data
05:26:42.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:26:42.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:26:42.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:26:42.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:26:42.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:26:42.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:26:42.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-0/logs
05:26:42.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-0/data/info.pb
05:26:42.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:26:42.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:26:42.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:26:42.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:26:42.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:26:42.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:26:42.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:26:42.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:26:42.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:26:42.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:26:42.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:26:42.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:26:42.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.254:45545
05:26:42.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.254
05:26:42.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40567
05:26:42.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/krb5kdc/kudu_127.1.19.254.keytab
05:26:42.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.254
05:26:42.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:26:42.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:26:42.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:26:42.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.212:36119
05:26:42.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:26:42.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:26:42.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:26:42.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:26:42.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.254:45545,127.1.19.253:39909,127.1.19.252:34617
05:26:42.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:26:42.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43121/jwks.json
05:26:42.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:26:42.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:42.532162 1391 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:26:42.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:42.532793 1391 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:26:42.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:42.533468 1391 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:26:42.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:42.565508 1391 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:26:42.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:42.565737 1391 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:26:42.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:42.566001 1391 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:26:42.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:42.566192 1391 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:26:42.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:42.566380 1391 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:26:42.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:42.566565 1391 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:26:42.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:42.566702 1391 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43121/jwks.json
05:26:42.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:42.604206 1391 master_runner.cc:386] Master server non-default flags:
05:26:42.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:26:42.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.212:36119
05:26:42.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:26:42.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:26:42.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-0/data
05:26:42.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-0/wal
05:26:42.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:26:42.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:26:42.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.254:45545,127.1.19.253:39909,127.1.19.252:34617
05:26:42.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/krb5kdc/kudu_127.1.19.254.keytab
05:26:42.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.254
05:26:42.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:26:42.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:26:42.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:26:42.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.254:45545
05:26:42.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:26:42.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:26:42.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:26:42.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43121/jwks.json
05:26:42.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:26:42.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:26:42.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:26:42.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:26:42.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:26:42.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-0/data/info.pb
05:26:42.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.254
05:26:42.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40567
05:26:42.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:26:42.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:26:42.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:26:42.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:26:42.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:26:42.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-0/logs
05:26:42.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:26:42.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:26:42.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:26:42.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:26:42.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:26:42.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:26:42.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:26:42.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 04:01:01 UTC on ccycloud.quasar-phqekz.root.comops.site
05:26:42.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679192
05:26:42.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:26:42.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:42.605297 1391 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:26:42.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:42.606225 1391 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:26:42.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:42.626372 1398 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:26:43.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:42.626462 1399 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:26:43.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:42.626392 1401 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:26:43.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:43.974216 1400 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
05:26:43.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:43.974334 1391 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
05:26:43.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:43.976071 1391 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:26:43.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:43.979084 1391 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:26:43.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:43.980515 1391 hybrid_clock.cc:648] HybridClock initialized: now 1790659603980477 us; error 52 us; skew 500 ppm
05:26:43.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:43.981305 1391 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:26:43.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:26:43 dist-test-slave-gvjk krb5kdc[1115](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790659603, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:26:43.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:43.987254 1391 init.cc:377] Logged in from keytab as kudu/127.1.19.254@KRBTEST.COM (short username kudu)
05:26:43.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:43.992357 1391 webserver.cc:460] Webserver started at http://127.1.19.254:40567/ using document root <none> and password file <none>
05:26:43.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:43.993238 1391 fs_manager.cc:359] Metadata directory not provided
05:26:43.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:43.993430 1391 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:26:44.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:44.001319 1391 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.000s sys 0.004s
05:26:44.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:44.005667 1410 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:26:44.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:44.006881 1391 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
05:26:44.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:44.007157 1391 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-0/data,/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-0/wal
05:26:44.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "82218c588ded4617b3da9bc194cc51c9"
05:26:44.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:26:37 on dist-test-slave-gvjk"
05:26:44.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:44.009500 1391 fs_report.cc:383] FS layout report
05:26:44.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:26:44.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-0/wal
05:26:44.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-0/wal
05:26:44.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-0/data/data
05:26:44.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:26:44.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:26:44.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:26:44.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:26:44.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:26:44.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:26:44.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:26:44.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:26:44.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:26:44.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:26:44.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:26:44.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:44.057400 1391 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:26:44.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:44.063525 1391 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:26:44.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:44.063982 1391 kserver.cc:163] Server-wide thread pool size limit: 3276
05:26:44.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:44.147392 1391 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.254:45545
05:26:44.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:44.147552 1461 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.254:45545 every 8 connection(s)
05:26:44.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:44.150540 1391 server_base.cc:1166] Dumped server information to /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-0/data/info.pb
05:26:44.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:44.151866 1103 external_mini_cluster.cc:1388] Started /tmp/dist-test-task_eGFIx/build/tsan/bin/kudu as pid 1391
05:26:44.152 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:26:44.152 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.19.252:34617
05:26:44.153 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
05:26:44.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:44.156049 1103 external_mini_cluster.cc:1326] Running /tmp/dist-test-task_eGFIx/build/tsan/bin/kudu
05:26:44.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_eGFIx/build/tsan/bin/kudu
05:26:44.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-2/wal
05:26:44.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-2/data
05:26:44.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:26:44.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:26:44.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:26:44.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:26:44.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:26:44.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:26:44.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-2/logs
05:26:44.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-2/data/info.pb
05:26:44.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:26:44.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:26:44.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:26:44.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:26:44.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:26:44.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:26:44.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:26:44.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:26:44.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:26:44.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:26:44.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:26:44.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:26:44.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.252:34617
05:26:44.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.252
05:26:44.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41891
05:26:44.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/krb5kdc/kudu_127.1.19.252.keytab
05:26:44.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.252
05:26:44.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:26:44.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:26:44.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:26:44.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.212:36119
05:26:44.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:26:44.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:26:44.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:26:44.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:26:44.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.254:45545,127.1.19.253:39909,127.1.19.252:34617
05:26:44.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:26:44.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43121/jwks.json
05:26:44.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:26:44.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:44.164968 1462 sys_catalog.cc:263] Verifying existing consensus state
05:26:44.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260929 05:26:44.165634 1462 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
05:26:44.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:44.166002 1391 master.cc:537] Master@127.1.19.254:45545 shutting down...
05:26:44.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:44.228844 1391 master.cc:552] Master@127.1.19.254:45545 shutdown complete.
05:26:44.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
05:26:44.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:44.489916 1464 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:26:44.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:44.490569 1464 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:26:44.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:44.491299 1464 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:26:44.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:44.523699 1464 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:26:44.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:44.524024 1464 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:26:44.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:44.524485 1464 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:26:44.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:44.524822 1464 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:26:44.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:44.525142 1464 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:26:44.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:44.525475 1464 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:26:44.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:44.525632 1464 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43121/jwks.json
05:26:44.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:44.563668 1464 master_runner.cc:386] Master server non-default flags:
05:26:44.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:26:44.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.212:36119
05:26:44.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:26:44.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:26:44.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-2/data
05:26:44.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-2/wal
05:26:44.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:26:44.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:26:44.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.254:45545,127.1.19.253:39909,127.1.19.252:34617
05:26:44.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/krb5kdc/kudu_127.1.19.252.keytab
05:26:44.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.252
05:26:44.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:26:44.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:26:44.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:26:44.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.252:34617
05:26:44.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:26:44.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:26:44.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:26:44.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43121/jwks.json
05:26:44.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:26:44.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:26:44.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:26:44.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:26:44.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:26:44.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-2/data/info.pb
05:26:44.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.252
05:26:44.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41891
05:26:44.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:26:44.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:26:44.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:26:44.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:26:44.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:26:44.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-2/logs
05:26:44.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:26:44.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:26:44.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:26:44.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:26:44.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:26:44.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:26:44.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:26:44.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 04:01:01 UTC on ccycloud.quasar-phqekz.root.comops.site
05:26:44.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679192
05:26:44.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:26:44.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:44.564814 1464 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:26:44.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:44.565804 1464 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:26:44.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:44.584487 1472 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:26:45.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:44.584636 1471 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:26:45.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:44.586701 1464 server_base.cc:1034] running on GCE node
05:26:45.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:44.584487 1474 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:26:45.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:45.899598 1464 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:26:45.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:45.903072 1464 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:26:45.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:45.904485 1464 hybrid_clock.cc:648] HybridClock initialized: now 1790659605904463 us; error 62 us; skew 500 ppm
05:26:45.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:45.905308 1464 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:26:45.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:26:45 dist-test-slave-gvjk krb5kdc[1115](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790659605, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:26:45.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:45.914752 1464 init.cc:377] Logged in from keytab as kudu/127.1.19.252@KRBTEST.COM (short username kudu)
05:26:45.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:45.919915 1464 webserver.cc:460] Webserver started at http://127.1.19.252:41891/ using document root <none> and password file <none>
05:26:45.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:45.920828 1464 fs_manager.cc:359] Metadata directory not provided
05:26:45.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:45.921021 1464 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:26:45.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:45.929266 1464 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.005s sys 0.003s
05:26:45.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:45.934317 1483 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:26:45.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:45.935463 1464 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.001s sys 0.004s
05:26:45.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:45.935734 1464 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-2/data,/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-2/wal
05:26:45.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "238e07129b03453d8f3bb2f71dfd1c9c"
05:26:45.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:26:41 on dist-test-slave-gvjk"
05:26:45.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:45.938033 1464 fs_report.cc:383] FS layout report
05:26:45.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:26:45.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-2/wal
05:26:45.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-2/wal
05:26:45.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-2/data/data
05:26:45.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:26:45.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:26:45.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:26:45.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:26:45.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:26:45.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:26:45.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:26:45.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:26:45.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:26:45.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:26:45.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:26:45.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:45.982020 1464 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:26:45.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:45.988664 1464 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:26:45.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:45.989086 1464 kserver.cc:163] Server-wide thread pool size limit: 3276
05:26:46.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:46.061295 1464 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.252:34617
05:26:46.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:46.061386 1534 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.252:34617 every 8 connection(s)
05:26:46.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:46.064388 1464 server_base.cc:1166] Dumped server information to /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-2/data/info.pb
05:26:46.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:46.074102 1103 external_mini_cluster.cc:1388] Started /tmp/dist-test-task_eGFIx/build/tsan/bin/kudu as pid 1464
05:26:46.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:46.075340 1535 sys_catalog.cc:263] Verifying existing consensus state
05:26:46.075 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:26:46.075 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.19.253:39909
05:26:46.076 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
05:26:46.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:46.077476 1103 external_mini_cluster.cc:1326] Running /tmp/dist-test-task_eGFIx/build/tsan/bin/kudu
05:26:46.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_eGFIx/build/tsan/bin/kudu
05:26:46.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-1/wal
05:26:46.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-1/data
05:26:46.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:26:46.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:26:46.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:26:46.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:26:46.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:26:46.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:26:46.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-1/logs
05:26:46.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-1/data/info.pb
05:26:46.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:26:46.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:26:46.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:26:46.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:26:46.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:26:46.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:26:46.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:26:46.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:26:46.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:26:46.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:26:46.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:26:46.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:26:46.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.253:39909
05:26:46.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.253
05:26:46.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42821
05:26:46.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/krb5kdc/kudu_127.1.19.253.keytab
05:26:46.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.253
05:26:46.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:26:46.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:26:46.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:26:46.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.212:36119
05:26:46.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:26:46.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:26:46.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:26:46.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:26:46.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.254:45545,127.1.19.253:39909,127.1.19.252:34617
05:26:46.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:26:46.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43121/jwks.json
05:26:46.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:26:46.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:46.081422 1535 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c: Bootstrap starting.
05:26:46.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:46.124284 1535 log.cc:824] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c: Log is configured to *not* fsync() on all Append() calls
05:26:46.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:46.129500 1535 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
05:26:46.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:46.130290 1535 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c: Bootstrap complete.
05:26:46.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:46.156816 1535 raft_consensus.cc:348] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82218c588ded4617b3da9bc194cc51c9" member_type: VOTER last_known_addr { host: "127.1.19.254" port: 45545 } } peers { permanent_uuid: "2fbfabd1811349149652cf260cc23e13" member_type: VOTER last_known_addr { host: "127.1.19.253" port: 39909 } } peers { permanent_uuid: "238e07129b03453d8f3bb2f71dfd1c9c" member_type: VOTER last_known_addr { host: "127.1.19.252" port: 34617 } }
05:26:46.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:46.159169 1535 raft_consensus.cc:374] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
05:26:46.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:46.159382 1535 raft_consensus.cc:729] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 238e07129b03453d8f3bb2f71dfd1c9c, State: Initialized, Role: FOLLOWER
05:26:46.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:46.160157 1535 consensus_queue.cc:260] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82218c588ded4617b3da9bc194cc51c9" member_type: VOTER last_known_addr { host: "127.1.19.254" port: 45545 } } peers { permanent_uuid: "2fbfabd1811349149652cf260cc23e13" member_type: VOTER last_known_addr { host: "127.1.19.253" port: 39909 } } peers { permanent_uuid: "238e07129b03453d8f3bb2f71dfd1c9c" member_type: VOTER last_known_addr { host: "127.1.19.252" port: 34617 } }
05:26:46.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:46.162547 1539 sys_catalog.cc:455] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82218c588ded4617b3da9bc194cc51c9" member_type: VOTER last_known_addr { host: "127.1.19.254" port: 45545 } } peers { permanent_uuid: "2fbfabd1811349149652cf260cc23e13" member_type: VOTER last_known_addr { host: "127.1.19.253" port: 39909 } } peers { permanent_uuid: "238e07129b03453d8f3bb2f71dfd1c9c" member_type: VOTER last_known_addr { host: "127.1.19.252" port: 34617 } } }
05:26:46.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:46.163458 1539 sys_catalog.cc:458] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [sys.catalog]: This master's current role is: FOLLOWER
05:26:46.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:46.163664 1535 sys_catalog.cc:564] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [sys.catalog]: configured and running, proceeding with master startup.
05:26:46.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:46.198097 1550 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:26:46.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:46.198426 1550 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:26:46.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:46.204528 1539 raft_consensus.cc:482] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:26:46.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:46.205042 1539 raft_consensus.cc:504] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82218c588ded4617b3da9bc194cc51c9" member_type: VOTER last_known_addr { host: "127.1.19.254" port: 45545 } } peers { permanent_uuid: "2fbfabd1811349149652cf260cc23e13" member_type: VOTER last_known_addr { host: "127.1.19.253" port: 39909 } } peers { permanent_uuid: "238e07129b03453d8f3bb2f71dfd1c9c" member_type: VOTER last_known_addr { host: "127.1.19.252" port: 34617 } }
05:26:46.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:46.209408 1485 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.254:45545: connect: Connection refused (error 111)
05:26:46.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:46.210672 1539 leader_election.cc:288] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82218c588ded4617b3da9bc194cc51c9 (127.1.19.254:45545), 2fbfabd1811349149652cf260cc23e13 (127.1.19.253:39909)
05:26:46.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:46.215171 1487 leader_election.cc:334] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2fbfabd1811349149652cf260cc23e13 (127.1.19.253:39909): Network error: Client connection negotiation failed: client connection to 127.1.19.253:39909: connect: Connection refused (error 111)
05:26:46.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:46.217231 1485 leader_election.cc:334] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 82218c588ded4617b3da9bc194cc51c9 (127.1.19.254:45545): Network error: Client connection negotiation failed: client connection to 127.1.19.254:45545: connect: Connection refused (error 111)
05:26:46.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:46.217634 1485 leader_election.cc:302] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 238e07129b03453d8f3bb2f71dfd1c9c; no voters: 2fbfabd1811349149652cf260cc23e13, 82218c588ded4617b3da9bc194cc51c9
05:26:46.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:46.218405 1539 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:26:46.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:46.188933 1464 master_runner.cc:186] Error getting master registration for 127.1.19.254:45545: Network error: Client connection negotiation failed: client connection to 127.1.19.254:45545: connect: Connection refused (error 111),
05:26:46.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:46.222469 1464 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:26:46.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:46.244625 1489 tablet.cc:2369] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c: Can't schedule compaction. Clean time has not been advanced past its initial value.
05:26:46.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:46.403900 1537 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:26:46.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:46.404614 1537 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:26:46.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:46.405298 1537 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:26:46.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:46.436822 1537 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:26:46.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:46.437039 1537 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:26:46.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:46.437338 1537 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:26:46.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:46.437538 1537 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:26:46.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:46.437721 1537 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:26:46.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:46.437901 1537 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:26:46.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:46.438035 1537 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43121/jwks.json
05:26:46.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:46.475871 1537 master_runner.cc:386] Master server non-default flags:
05:26:46.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:26:46.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.212:36119
05:26:46.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:26:46.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:26:46.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-1/data
05:26:46.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-1/wal
05:26:46.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:26:46.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:26:46.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.254:45545,127.1.19.253:39909,127.1.19.252:34617
05:26:46.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/krb5kdc/kudu_127.1.19.253.keytab
05:26:46.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.253
05:26:46.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:26:46.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:26:46.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:26:46.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.253:39909
05:26:46.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:26:46.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:26:46.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:26:46.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43121/jwks.json
05:26:46.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:26:46.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:26:46.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:26:46.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:26:46.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:26:46.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-1/data/info.pb
05:26:46.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.253
05:26:46.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42821
05:26:46.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:26:46.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:26:46.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:26:46.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:26:46.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:26:46.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-1/logs
05:26:46.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:26:46.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:26:46.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:26:46.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:26:46.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:26:46.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:26:46.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:26:46.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 04:01:01 UTC on ccycloud.quasar-phqekz.root.comops.site
05:26:46.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679192
05:26:46.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:26:46.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:46.477113 1537 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:26:46.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:46.478278 1537 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:26:46.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:46.509722 1561 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:26:47.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:47.203188 1550 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:26:47.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:47.203889 1550 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:26:47.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:47.987283 1567 raft_consensus.cc:482] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:26:47.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:47.988369 1567 raft_consensus.cc:504] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82218c588ded4617b3da9bc194cc51c9" member_type: VOTER last_known_addr { host: "127.1.19.254" port: 45545 } } peers { permanent_uuid: "2fbfabd1811349149652cf260cc23e13" member_type: VOTER last_known_addr { host: "127.1.19.253" port: 39909 } } peers { permanent_uuid: "238e07129b03453d8f3bb2f71dfd1c9c" member_type: VOTER last_known_addr { host: "127.1.19.252" port: 34617 } }
05:26:48.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:48.013655 1567 leader_election.cc:288] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82218c588ded4617b3da9bc194cc51c9 (127.1.19.254:45545), 2fbfabd1811349149652cf260cc23e13 (127.1.19.253:39909)
05:26:48.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:48.042481 1485 leader_election.cc:334] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 82218c588ded4617b3da9bc194cc51c9 (127.1.19.254:45545): Network error: Client connection negotiation failed: client connection to 127.1.19.254:45545: connect: Connection refused (error 111)
05:26:48.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:48.049479 1487 leader_election.cc:334] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2fbfabd1811349149652cf260cc23e13 (127.1.19.253:39909): Network error: Client connection negotiation failed: client connection to 127.1.19.253:39909: connect: Connection refused (error 111)
05:26:48.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:48.050498 1487 leader_election.cc:302] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 238e07129b03453d8f3bb2f71dfd1c9c; no voters: 2fbfabd1811349149652cf260cc23e13, 82218c588ded4617b3da9bc194cc51c9
05:26:48.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:48.051875 1567 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:26:48.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:48.208729 1550 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:26:48.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:48.209092 1550 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:26:48.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:47.902354 1560 debug-util.cc:398] Leaking SignalData structure 0x7b0800014920 after lost signal to thread 1537
05:26:48.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:48.867614 1560 kernel_stack_watchdog.cc:198] Thread 1537 stuck at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:637 for 388ms:
05:26:48.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
05:26:48.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
05:26:48.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
05:26:48.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
05:26:48.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:48.870200 1564 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:26:48.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:48.872916 1537 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 2.361s user 0.681s sys 1.210s
05:26:48.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:48.873253 1537 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 2.362s user 0.681s sys 1.210s
05:26:48.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:46.511389 1562 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:26:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:48.877398 1563 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2364 milliseconds
05:26:48.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:48.877430 1537 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
05:26:48.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:48.878875 1537 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:26:48.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:48.882156 1537 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:26:48.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:48.883524 1537 hybrid_clock.cc:648] HybridClock initialized: now 1790659608883482 us; error 50 us; skew 500 ppm
05:26:48.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:48.884346 1537 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:26:48.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:26:48 dist-test-slave-gvjk krb5kdc[1115](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790659608, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:26:48.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:48.891115 1537 init.cc:377] Logged in from keytab as kudu/127.1.19.253@KRBTEST.COM (short username kudu)
05:26:48.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:48.896744 1537 webserver.cc:460] Webserver started at http://127.1.19.253:42821/ using document root <none> and password file <none>
05:26:48.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:48.897909 1537 fs_manager.cc:359] Metadata directory not provided
05:26:48.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:48.898177 1537 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:26:48.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:48.909279 1537 fs_manager.cc:709] Time spent opening directory manager: real 0.007s user 0.002s sys 0.006s
05:26:48.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:48.914984 1576 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:26:48.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:48.916365 1537 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.003s
05:26:48.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:48.916760 1537 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-1/data,/tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-1/wal
05:26:48.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2fbfabd1811349149652cf260cc23e13"
05:26:48.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:26:39 on dist-test-slave-gvjk"
05:26:48.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:48.920089 1537 fs_report.cc:383] FS layout report
05:26:48.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:26:48.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-1/wal
05:26:48.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-1/wal
05:26:48.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-1/data/data
05:26:48.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:26:48.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:26:48.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:26:48.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:26:48.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:26:48.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:26:48.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:26:48.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:26:48.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:26:48.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:26:48.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:26:48.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:48.976568 1537 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:26:48.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:48.984184 1537 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:26:48.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:48.984632 1537 kserver.cc:163] Server-wide thread pool size limit: 3276
05:26:49.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:49.097513 1537 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.253:39909
05:26:49.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:49.097666 1627 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.253:39909 every 8 connection(s)
05:26:49.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:49.100426 1537 server_base.cc:1166] Dumped server information to /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-1/data/info.pb
05:26:49.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:49.105250 1103 external_mini_cluster.cc:1388] Started /tmp/dist-test-task_eGFIx/build/tsan/bin/kudu as pid 1537
05:26:49.106 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:26:49.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:49.115471 1628 sys_catalog.cc:263] Verifying existing consensus state
05:26:49.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260929 05:26:49.116606 1628 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
05:26:49.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:49.117141 1537 master.cc:537] Master@127.1.19.253:39909 shutting down...
05:26:49.202 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.19.254:45545(127.1.19.254:45545)] connecting to peer
05:26:49.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:49.210512 1537 master.cc:552] Master@127.1.19.253:39909 shutdown complete.
05:26:49.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:49.215670 1550 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:26:49.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:49.216975 1550 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:26:49.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task_eGFIx/test-tmp/mini-kudu-cluster7731141829199071732/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
05:26:49.285 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.19.252:34617(127.1.19.252:34617)] connecting to peer
05:26:49.294 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.19.253:39909(127.1.19.253:39909)] connecting to peer
05:26:49.349 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.1.19.252:34617(127.1.19.252:34617)] Successfully connected to peer
05:26:49.354 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:49.356 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.254:45545(127.1.19.254:45545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:49.358 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45545: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:49.359 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:49.359 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:39909(127.1.19.253:39909)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:49.360 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:39909: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:49.428 [WARN - kudu-nio-1] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
05:26:49.437 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.1.19.252
05:26:49.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:26:49 dist-test-slave-gvjk krb5kdc[1115](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790659602, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.19.252@KRBTEST.COM
05:26:49.702 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xdfd702e1, L:/127.0.0.1:56206 - R:/127.1.19.252:34617] using SASL/GSSAPI
05:26:49.717 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909; will retry
05:26:49.734 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45545(127.1.19.254:45545)] connecting to peer
05:26:49.738 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:39909(127.1.19.253:39909)] connecting to peer
05:26:49.741 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:49.741 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:39909(127.1.19.253:39909)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:49.741 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:39909: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:49.742 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:49.742 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45545(127.1.19.254:45545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:49.743 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45545: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:49.745 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909; will retry
05:26:49.752 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45545(127.1.19.254:45545)] connecting to peer
05:26:49.754 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:39909(127.1.19.253:39909)] connecting to peer
05:26:49.757 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:49.757 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:49.757 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45545(127.1.19.254:45545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:49.757 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:39909(127.1.19.253:39909)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:49.758 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45545: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:49.758 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:39909: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:49.760 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909; will retry
05:26:49.772 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45545(127.1.19.254:45545)] connecting to peer
05:26:49.774 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:39909(127.1.19.253:39909)] connecting to peer
05:26:49.777 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:49.777 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:39909(127.1.19.253:39909)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:49.778 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:39909: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:49.779 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:49.779 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45545(127.1.19.254:45545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:49.781 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45545: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:49.781 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909; will retry
05:26:49.793 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45545(127.1.19.254:45545)] connecting to peer
05:26:49.794 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:39909(127.1.19.253:39909)] connecting to peer
05:26:49.796 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:49.796 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:39909(127.1.19.253:39909)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:49.797 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:39909: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:49.797 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:49.797 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45545(127.1.19.254:45545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:49.798 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45545: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:49.800 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909; will retry
05:26:49.832 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45545(127.1.19.254:45545)] connecting to peer
05:26:49.834 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:39909(127.1.19.253:39909)] connecting to peer
05:26:49.836 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:49.836 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:39909(127.1.19.253:39909)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:49.837 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:39909: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:49.839 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:49.840 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45545(127.1.19.254:45545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:49.840 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45545: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:49.842 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909; will retry
05:26:49.872 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45545(127.1.19.254:45545)] connecting to peer
05:26:49.874 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:39909(127.1.19.253:39909)] connecting to peer
05:26:49.876 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:49.877 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:49.877 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:39909(127.1.19.253:39909)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:49.877 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45545(127.1.19.254:45545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:49.877 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:39909: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:49.878 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45545: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:49.879 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909; will retry
05:26:50.012 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45545(127.1.19.254:45545)] connecting to peer
05:26:50.014 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:39909(127.1.19.253:39909)] connecting to peer
05:26:50.016 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:50.016 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:50.016 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:39909(127.1.19.253:39909)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:50.017 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45545(127.1.19.254:45545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:50.017 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:39909: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:50.017 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45545: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:50.020 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909; will retry
05:26:50.173 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45545(127.1.19.254:45545)] connecting to peer
05:26:50.175 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:39909(127.1.19.253:39909)] connecting to peer
05:26:50.177 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:50.177 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:50.177 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45545(127.1.19.254:45545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:50.177 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:39909(127.1.19.253:39909)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:50.179 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:39909: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:50.179 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45545: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:50.182 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909; will retry
05:26:50.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:50.220206 1550 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:26:50.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:50.220501 1550 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:26:50.312 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45545(127.1.19.254:45545)] connecting to peer
05:26:50.314 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:39909(127.1.19.253:39909)] connecting to peer
05:26:50.316 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:50.316 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:39909(127.1.19.253:39909)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:50.316 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:50.316 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45545(127.1.19.254:45545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:50.317 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:39909: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:50.318 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45545: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:50.319 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909; will retry
05:26:50.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:50.375521 1638 raft_consensus.cc:482] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:26:50.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:50.376047 1638 raft_consensus.cc:504] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82218c588ded4617b3da9bc194cc51c9" member_type: VOTER last_known_addr { host: "127.1.19.254" port: 45545 } } peers { permanent_uuid: "2fbfabd1811349149652cf260cc23e13" member_type: VOTER last_known_addr { host: "127.1.19.253" port: 39909 } } peers { permanent_uuid: "238e07129b03453d8f3bb2f71dfd1c9c" member_type: VOTER last_known_addr { host: "127.1.19.252" port: 34617 } }
05:26:50.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:50.377794 1638 leader_election.cc:288] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82218c588ded4617b3da9bc194cc51c9 (127.1.19.254:45545), 2fbfabd1811349149652cf260cc23e13 (127.1.19.253:39909)
05:26:50.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:50.383880 1485 leader_election.cc:334] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 82218c588ded4617b3da9bc194cc51c9 (127.1.19.254:45545): Network error: Client connection negotiation failed: client connection to 127.1.19.254:45545: connect: Connection refused (error 111)
05:26:50.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:50.384579 1487 leader_election.cc:334] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2fbfabd1811349149652cf260cc23e13 (127.1.19.253:39909): Network error: Client connection negotiation failed: client connection to 127.1.19.253:39909: connect: Connection refused (error 111)
05:26:50.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:50.384989 1487 leader_election.cc:302] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 238e07129b03453d8f3bb2f71dfd1c9c; no voters: 2fbfabd1811349149652cf260cc23e13, 82218c588ded4617b3da9bc194cc51c9
05:26:50.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:50.385673 1638 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:26:50.733 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45545(127.1.19.254:45545)] connecting to peer
05:26:50.734 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:39909(127.1.19.253:39909)] connecting to peer
05:26:50.736 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:50.736 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:39909(127.1.19.253:39909)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:50.736 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:39909: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:50.737 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:50.737 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45545(127.1.19.254:45545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:50.737 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45545: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:50.738 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909; will retry
05:26:50.772 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45545(127.1.19.254:45545)] connecting to peer
05:26:50.774 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:39909(127.1.19.253:39909)] connecting to peer
05:26:50.777 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:50.777 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:39909(127.1.19.253:39909)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:50.777 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:39909: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:50.778 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:50.778 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45545(127.1.19.254:45545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:50.779 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45545: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:50.779 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909; will retry
05:26:51.193 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45545(127.1.19.254:45545)] connecting to peer
05:26:51.194 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:39909(127.1.19.253:39909)] connecting to peer
05:26:51.196 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:51.196 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:39909(127.1.19.253:39909)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:51.196 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:51.196 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:39909: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:51.196 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45545(127.1.19.254:45545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:51.197 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45545: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:51.198 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909; will retry
05:26:51.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:51.223590 1550 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:26:51.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:51.223914 1550 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:26:51.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:51.225626 1464 master_runner.cc:186] Error getting master registration for 127.1.19.254:45545: Network error: Client connection negotiation failed: client connection to 127.1.19.254:45545: connect: Connection refused (error 111),
05:26:51.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:51.226019 1464 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:26:51.412 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45545(127.1.19.254:45545)] connecting to peer
05:26:51.413 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:39909(127.1.19.253:39909)] connecting to peer
05:26:51.415 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:51.416 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:39909(127.1.19.253:39909)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:51.416 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:39909: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:51.416 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:51.416 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45545(127.1.19.254:45545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:51.417 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45545: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:51.418 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909; will retry
05:26:52.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:52.226660 1550 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:26:52.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:52.227056 1550 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:26:53.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:53.230346 1550 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:26:53.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:53.230645 1550 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:26:53.792 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45545(127.1.19.254:45545)] connecting to peer
05:26:53.794 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:39909(127.1.19.253:39909)] connecting to peer
05:26:53.796 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:53.796 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:53.796 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45545(127.1.19.254:45545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:53.797 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:39909(127.1.19.253:39909)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:53.797 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45545: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:53.797 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:39909: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:53.799 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909; will retry
05:26:54.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:54.111909 1648 raft_consensus.cc:482] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:26:54.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:54.112478 1648 raft_consensus.cc:504] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82218c588ded4617b3da9bc194cc51c9" member_type: VOTER last_known_addr { host: "127.1.19.254" port: 45545 } } peers { permanent_uuid: "2fbfabd1811349149652cf260cc23e13" member_type: VOTER last_known_addr { host: "127.1.19.253" port: 39909 } } peers { permanent_uuid: "238e07129b03453d8f3bb2f71dfd1c9c" member_type: VOTER last_known_addr { host: "127.1.19.252" port: 34617 } }
05:26:54.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:54.114307 1648 leader_election.cc:288] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82218c588ded4617b3da9bc194cc51c9 (127.1.19.254:45545), 2fbfabd1811349149652cf260cc23e13 (127.1.19.253:39909)
05:26:54.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:54.117481 1485 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.254:45545: connect: Connection refused (error 111) [suppressed 5 similar messages]
05:26:54.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:54.120359 1487 leader_election.cc:334] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2fbfabd1811349149652cf260cc23e13 (127.1.19.253:39909): Network error: Client connection negotiation failed: client connection to 127.1.19.253:39909: connect: Connection refused (error 111)
05:26:54.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:54.120851 1485 leader_election.cc:334] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 82218c588ded4617b3da9bc194cc51c9 (127.1.19.254:45545): Network error: Client connection negotiation failed: client connection to 127.1.19.254:45545: connect: Connection refused (error 111)
05:26:54.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:54.121236 1485 leader_election.cc:302] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 238e07129b03453d8f3bb2f71dfd1c9c; no voters: 2fbfabd1811349149652cf260cc23e13, 82218c588ded4617b3da9bc194cc51c9
05:26:54.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:54.121942 1648 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:26:54.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:54.234097 1550 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:26:54.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:54.234365 1550 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:26:55.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:55.238549 1550 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:26:55.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:55.238889 1550 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:26:55.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:55.706137 1088 debug-util.cc:398] Leaking SignalData structure 0x7b08000bb280 after lost signal to thread 963
05:26:55.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:55.707006 1088 debug-util.cc:398] Leaking SignalData structure 0x7b08000af8e0 after lost signal to thread 1091
05:26:55.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:26:55 dist-test-slave-gvjk krb5kdc[1115](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790659615, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:26:55.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:55.918733 1479 init.cc:295] Successfully reacquired a new kerberos TGT
05:26:56.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:56.229929 1464 master_runner.cc:186] Error getting master registration for 127.1.19.254:45545: Network error: Client connection negotiation failed: client connection to 127.1.19.254:45545: connect: Connection refused (error 111),
05:26:56.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:56.230458 1464 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:26:56.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:56.241928 1550 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:26:56.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:56.242238 1550 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:26:57.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:57.245383 1550 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:26:57.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:57.245685 1550 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:26:57.512 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45545(127.1.19.254:45545)] connecting to peer
05:26:57.514 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:39909(127.1.19.253:39909)] connecting to peer
05:26:57.517 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:57.517 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:57.517 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45545(127.1.19.254:45545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:57.517 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:39909(127.1.19.253:39909)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:57.519 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45545: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:57.519 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:39909: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:57.520 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909; will retry
05:26:58.152 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45545(127.1.19.254:45545)] connecting to peer
05:26:58.154 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:39909(127.1.19.253:39909)] connecting to peer
05:26:58.156 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:58.156 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:58.157 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45545(127.1.19.254:45545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:58.157 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:39909(127.1.19.253:39909)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:58.157 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:39909: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:58.157 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45545: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:58.158 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909; will retry
05:26:58.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:58.248821 1550 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:26:58.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:58.249104 1550 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:26:58.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:58.848834 1656 raft_consensus.cc:482] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:26:58.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:58.849273 1656 raft_consensus.cc:504] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82218c588ded4617b3da9bc194cc51c9" member_type: VOTER last_known_addr { host: "127.1.19.254" port: 45545 } } peers { permanent_uuid: "2fbfabd1811349149652cf260cc23e13" member_type: VOTER last_known_addr { host: "127.1.19.253" port: 39909 } } peers { permanent_uuid: "238e07129b03453d8f3bb2f71dfd1c9c" member_type: VOTER last_known_addr { host: "127.1.19.252" port: 34617 } }
05:26:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:58.850839 1656 leader_election.cc:288] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82218c588ded4617b3da9bc194cc51c9 (127.1.19.254:45545), 2fbfabd1811349149652cf260cc23e13 (127.1.19.253:39909)
05:26:58.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:58.856204 1485 leader_election.cc:334] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 82218c588ded4617b3da9bc194cc51c9 (127.1.19.254:45545): Network error: Client connection negotiation failed: client connection to 127.1.19.254:45545: connect: Connection refused (error 111)
05:26:58.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:58.856992 1487 leader_election.cc:334] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2fbfabd1811349149652cf260cc23e13 (127.1.19.253:39909): Network error: Client connection negotiation failed: client connection to 127.1.19.253:39909: connect: Connection refused (error 111)
05:26:58.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:58.857450 1487 leader_election.cc:302] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 238e07129b03453d8f3bb2f71dfd1c9c; no voters: 2fbfabd1811349149652cf260cc23e13, 82218c588ded4617b3da9bc194cc51c9
05:26:58.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:26:58.858229 1656 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:26:59.213 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45545(127.1.19.254:45545)] connecting to peer
05:26:59.214 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:39909(127.1.19.253:39909)] connecting to peer
05:26:59.216 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:59.217 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45545(127.1.19.254:45545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:59.217 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45545: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:59.218 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:59.218 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:39909(127.1.19.253:39909)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:59.218 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:39909: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:59.219 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909; will retry
05:26:59.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:59.252050 1550 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:26:59.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:26:59.252298 1550 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:26:59.852 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45545(127.1.19.254:45545)] connecting to peer
05:26:59.855 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:39909(127.1.19.253:39909)] connecting to peer
05:26:59.856 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:59.857 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45545(127.1.19.254:45545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:59.857 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:59.857 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:39909(127.1.19.253:39909)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:59.857 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45545: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:26:59.857 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:39909: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:26:59.858 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909; will retry
05:27:00.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:27:00.255548 1550 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:27:00.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:27:00.255831 1550 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:27:01.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:27:01.233526 1464 master_runner.cc:186] Error getting master registration for 127.1.19.254:45545: Network error: Client connection negotiation failed: client connection to 127.1.19.254:45545: connect: Connection refused (error 111),
05:27:01.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:27:01.233983 1464 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:27:01.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:27:01.259142 1550 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:27:01.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:27:01.259447 1550 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:27:01.473 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45545(127.1.19.254:45545)] connecting to peer
05:27:01.474 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:39909(127.1.19.253:39909)] connecting to peer
05:27:01.476 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:27:01.476 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:27:01.476 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45545(127.1.19.254:45545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:27:01.476 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:39909(127.1.19.253:39909)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:27:01.476 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:39909: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:27:01.476 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45545: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:27:01.478 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909; will retry
05:27:01.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:27:01.876884 1666 raft_consensus.cc:482] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:27:01.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:27:01.877249 1666 raft_consensus.cc:504] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82218c588ded4617b3da9bc194cc51c9" member_type: VOTER last_known_addr { host: "127.1.19.254" port: 45545 } } peers { permanent_uuid: "2fbfabd1811349149652cf260cc23e13" member_type: VOTER last_known_addr { host: "127.1.19.253" port: 39909 } } peers { permanent_uuid: "238e07129b03453d8f3bb2f71dfd1c9c" member_type: VOTER last_known_addr { host: "127.1.19.252" port: 34617 } }
05:27:01.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:27:01.878757 1666 leader_election.cc:288] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82218c588ded4617b3da9bc194cc51c9 (127.1.19.254:45545), 2fbfabd1811349149652cf260cc23e13 (127.1.19.253:39909)
05:27:01.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:27:01.881294 1487 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.253:39909: connect: Connection refused (error 111) [suppressed 3 similar messages]
05:27:01.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:27:01.883713 1485 leader_election.cc:334] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 82218c588ded4617b3da9bc194cc51c9 (127.1.19.254:45545): Network error: Client connection negotiation failed: client connection to 127.1.19.254:45545: connect: Connection refused (error 111)
05:27:01.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:27:01.884178 1487 leader_election.cc:334] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2fbfabd1811349149652cf260cc23e13 (127.1.19.253:39909): Network error: Client connection negotiation failed: client connection to 127.1.19.253:39909: connect: Connection refused (error 111)
05:27:01.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:27:01.884624 1487 leader_election.cc:302] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 238e07129b03453d8f3bb2f71dfd1c9c; no voters: 2fbfabd1811349149652cf260cc23e13, 82218c588ded4617b3da9bc194cc51c9
05:27:01.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:27:01.885284 1666 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:27:02.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:27:02.262233 1550 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:27:02.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:27:02.262534 1550 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:27:02.292 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45545(127.1.19.254:45545)] connecting to peer
05:27:02.293 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:39909(127.1.19.253:39909)] connecting to peer
05:27:02.294 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:27:02.294 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45545(127.1.19.254:45545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:27:02.294 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45545: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:27:02.295 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:27:02.295 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:39909(127.1.19.253:39909)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:27:02.295 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:39909: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:27:02.298 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909; will retry
05:27:02.512 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45545(127.1.19.254:45545)] connecting to peer
05:27:02.514 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:39909(127.1.19.253:39909)] connecting to peer
05:27:02.514 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:27:02.514 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:27:02.515 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:39909(127.1.19.253:39909)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:27:02.515 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45545(127.1.19.254:45545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:27:02.515 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:39909: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:27:02.515 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45545: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:27:02.517 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909; will retry
05:27:03.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:27:03.265442 1550 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:27:03.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:27:03.265746 1550 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:27:04.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:27:04.268949 1550 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:27:04.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:27:04.269313 1550 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:27:05.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:27:05.273582 1550 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:27:05.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:27:05.273983 1550 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:27:05.612 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45545(127.1.19.254:45545)] connecting to peer
05:27:05.613 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:39909(127.1.19.253:39909)] connecting to peer
05:27:05.614 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:27:05.614 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:39909(127.1.19.253:39909)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:27:05.614 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:39909: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:27:05.614 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:27:05.615 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45545(127.1.19.254:45545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:27:05.615 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45545: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:27:05.617 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909; will retry
05:27:05.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:27:05 dist-test-slave-gvjk krb5kdc[1115](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790659625, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:27:05.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:27:05.921128 1479 init.cc:295] Successfully reacquired a new kerberos TGT
05:27:06.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:27:06.237560 1464 master_runner.cc:186] Error getting master registration for 127.1.19.254:45545: Network error: Client connection negotiation failed: client connection to 127.1.19.254:45545: connect: Connection refused (error 111),
05:27:06.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:27:06.237994 1464 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:27:06.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:27:06.277415 1550 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:27:06.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:27:06.277745 1550 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:27:07.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:27:07.281658 1550 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:27:07.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:27:07.282003 1550 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:27:08.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:27:08.286357 1550 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:27:08.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:27:08.286782 1550 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:27:08.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:27:08.973816 1677 raft_consensus.cc:482] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:27:08.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:27:08.974356 1677 raft_consensus.cc:504] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82218c588ded4617b3da9bc194cc51c9" member_type: VOTER last_known_addr { host: "127.1.19.254" port: 45545 } } peers { permanent_uuid: "2fbfabd1811349149652cf260cc23e13" member_type: VOTER last_known_addr { host: "127.1.19.253" port: 39909 } } peers { permanent_uuid: "238e07129b03453d8f3bb2f71dfd1c9c" member_type: VOTER last_known_addr { host: "127.1.19.252" port: 34617 } }
05:27:08.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:27:08.976675 1677 leader_election.cc:288] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82218c588ded4617b3da9bc194cc51c9 (127.1.19.254:45545), 2fbfabd1811349149652cf260cc23e13 (127.1.19.253:39909)
05:27:08.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:27:08.978978 1485 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.254:45545: connect: Connection refused (error 111) [suppressed 1 similar messages]
05:27:08.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:27:08.981562 1487 leader_election.cc:334] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2fbfabd1811349149652cf260cc23e13 (127.1.19.253:39909): Network error: Client connection negotiation failed: client connection to 127.1.19.253:39909: connect: Connection refused (error 111)
05:27:08.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:27:08.982044 1485 leader_election.cc:334] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 82218c588ded4617b3da9bc194cc51c9 (127.1.19.254:45545): Network error: Client connection negotiation failed: client connection to 127.1.19.254:45545: connect: Connection refused (error 111)
05:27:08.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:27:08.982365 1485 leader_election.cc:302] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 238e07129b03453d8f3bb2f71dfd1c9c; no voters: 2fbfabd1811349149652cf260cc23e13, 82218c588ded4617b3da9bc194cc51c9
05:27:08.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:27:08.982975 1677 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:27:09.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:27:09.290755 1550 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:27:09.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:27:09.291085 1550 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:27:09.452 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45545(127.1.19.254:45545)] connecting to peer
05:27:09.453 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:39909(127.1.19.253:39909)] connecting to peer
05:27:09.454 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:27:09.454 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:27:09.454 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45545(127.1.19.254:45545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:27:09.455 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45545: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:27:09.454 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:39909(127.1.19.253:39909)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:27:09.455 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:39909: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:27:09.458 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909; will retry
05:27:10.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:27:10.295495 1550 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:27:10.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:27:10.295903 1550 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:27:10.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:27:10.856683 1088 debug-util.cc:398] Leaking SignalData structure 0x7b08000bb680 after lost signal to thread 963
05:27:10.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:27:10.857723 1088 debug-util.cc:398] Leaking SignalData structure 0x7b08000bbfa0 after lost signal to thread 1091
05:27:11.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:27:11.240720 1464 master_runner.cc:186] Error getting master registration for 127.1.19.254:45545: Network error: Client connection negotiation failed: client connection to 127.1.19.254:45545: connect: Connection refused (error 111),
05:27:11.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:27:11.241143 1464 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:27:11.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:27:11.299891 1550 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:27:11.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:27:11.300227 1550 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:27:12.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:27:12.304139 1550 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:27:12.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:27:12.304478 1550 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:27:13.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:27:13.307636 1550 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:27:13.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:27:13.307933 1550 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:27:13.532 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45545(127.1.19.254:45545)] connecting to peer
05:27:13.533 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:39909(127.1.19.253:39909)] connecting to peer
05:27:13.534 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:27:13.534 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:27:13.535 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:39909(127.1.19.253:39909)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:27:13.535 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45545(127.1.19.254:45545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:27:13.535 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:39909: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:27:13.535 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45545: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:27:13.537 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909; will retry
05:27:13.592 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45545(127.1.19.254:45545)] connecting to peer
05:27:13.593 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:39909(127.1.19.253:39909)] connecting to peer
05:27:13.594 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:27:13.594 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:39909(127.1.19.253:39909)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:27:13.594 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:39909: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:27:13.594 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:27:13.594 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45545(127.1.19.254:45545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:27:13.595 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45545: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:27:13.596 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909; will retry
05:27:14.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:27:14.311444 1550 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:27:14.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:27:14.311713 1550 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:27:14.792 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45545(127.1.19.254:45545)] connecting to peer
05:27:14.793 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:39909(127.1.19.253:39909)] connecting to peer
05:27:14.794 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:27:14.794 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:39909(127.1.19.253:39909)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:27:14.794 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:27:14.794 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45545(127.1.19.254:45545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:27:14.794 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:39909: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:27:14.794 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45545: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:27:14.798 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909; will retry
05:27:15.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:27:15.315239 1550 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:27:15.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:27:15.315697 1550 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:27:15.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:27:15 dist-test-slave-gvjk krb5kdc[1115](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790659635, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:27:15.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:27:15.924300 1479 init.cc:295] Successfully reacquired a new kerberos TGT
05:27:16.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:27:16.244612 1464 master_runner.cc:186] Error getting master registration for 127.1.19.254:45545: Network error: Client connection negotiation failed: client connection to 127.1.19.254:45545: connect: Connection refused (error 111),
05:27:16.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:27:16.245067 1464 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:27:16.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:27:16.319043 1550 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:27:16.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:27:16.319327 1550 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:27:17.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:27:17.321991 1550 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:27:17.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:27:17.322288 1550 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:27:18.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:27:18.324785 1550 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 238e07129b03453d8f3bb2f71dfd1c9c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:27:18.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:27:18.325085 1550 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:27:18.893 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.254:45545(127.1.19.254:45545)] connecting to peer
05:27:18.894 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.253:39909(127.1.19.253:39909)] connecting to peer
05:27:18.895 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:27:18.895 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:27:18.895 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.253:39909(127.1.19.253:39909)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:27:18.895 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.254:45545(127.1.19.254:45545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:27:18.895 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.253:39909: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909
05:27:18.895 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.254:45545: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
05:27:18.897 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909; will retry
05:27:18.901 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=28, TimeoutTracker(timeout=30000, elapsed=29787), Traces: [0ms] refreshing cache from master, [76ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [168ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [177ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [240ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [243ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [597ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [601ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [616ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [622ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [625ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [626ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [628ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [629ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [636ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [641ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [641ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [643ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [644ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [661ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [662ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [664ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [665ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [676ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [681ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [682ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [683ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [684ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [716ms] refreshing cache from master, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [720ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [724ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [726ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [726ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [755ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [761ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [761ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [763ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [764ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [896ms] refreshing cache from master, [896ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [901ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [901ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [903ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [904ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [1056ms] refreshing cache from master, [1057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [1058ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [1059ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [1062ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [1062ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [1064ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [1066ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [1195ms] refreshing cache from master, [1196ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [1196ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [1198ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [1200ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [1201ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [1203ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [1204ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [1616ms] refreshing cache from master, [1616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [1617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [1618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [1620ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [1621ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [1622ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [1623ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [1656ms] refreshing cache from master, [1656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [1658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [1658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [1661ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [1662ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [1663ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [1663ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [2076ms] refreshing cache from master, [2077ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [2077ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [2078ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [2080ms] too many traces: truncated at 100 traces, deferred=Deferred@99617476(state=PENDING, result=null, callback=(continuation of Deferred@79953651 after retry RPC after error@409222695) -> (continuation of Deferred@90765087 after retry RPC after error@13335388) -> (continuation of Deferred@467937744 after retry RPC after error@575191437) -> (continuation of Deferred@1624924884 after retry RPC after error@999351304) -> (continuation of Deferred@16834563 after retry RPC after error@1726199167) -> (continuation of Deferred@1436334005 after retry RPC after error@1771092394) -> (continuation of Deferred@1872120931 after retry RPC after error@346375331) -> (continuation of Deferred@615807133 after retry RPC after error@1208338504) -> (continuation of Deferred@355299455 after retry RPC after error@731098661) -> (continuation of Deferred@744770394 after retry RPC after error@1254461718) -> (continuation of Deferred@873840313 after retry RPC after error@37870744) -> (continuation of Deferred@217147665 after retry RPC after error@1792672952) -> (continuation of Deferred@1940102956 after retry RPC after error@2139764956) -> (continuation of Deferred@2025091037 after retry RPC after error@1123109815) -> (continuation of Deferred@759490106 after retry RPC after error@52848534) -> (continuation of Deferred@1488403112 after retry RPC after error@282296058) -> (continuation of Deferred@439107564 after retry RPC after error@713482728) -> (continuation of Deferred@1267072106 after retry RPC after error@1076526926) -> (continuation of Deferred@990047224 after retry RPC after error@647414993) -> (continuation of Deferred@1116998341 after retry RPC after error@1373101526) -> (continuation of Deferred@2108076351 after retry RPC after error@1837519382) -> (continuation of Deferred@262586177 after retry RPC after error@1626405390) -> (continuation of Deferred@672628838 after retry RPC after error@2002678160) -> (continuation of Deferred@1303942285 after retry RPC after error@721364926) -> (continuation of Deferred@1048548919 after retry RPC after error@975811225) -> (continuation of Deferred@859588013 after retry RPC after error@1621968547) -> (continuation of Deferred@581024431 after retry RPC after error@1441495320), errback=(continuation of Deferred@79953651 after retry RPC after error@409222695) -> (continuation of Deferred@90765087 after retry RPC after error@13335388) -> (continuation of Deferred@467937744 after retry RPC after error@575191437) -> (continuation of Deferred@1624924884 after retry RPC after error@999351304) -> (continuation of Deferred@16834563 after retry RPC after error@1726199167) -> (continuation of Deferred@1436334005 after retry RPC after error@1771092394) -> (continuation of Deferred@1872120931 after retry RPC after error@346375331) -> (continuation of Deferred@615807133 after retry RPC after error@1208338504) -> (continuation of Deferred@355299455 after retry RPC after error@731098661) -> (continuation of Deferred@744770394 after retry RPC after error@1254461718) -> (continuation of Deferred@873840313 after retry RPC after error@37870744) -> (continuation of Deferred@217147665 after retry RPC after error@1792672952) -> (continuation of Deferred@1940102956 after retry RPC after error@2139764956) -> (continuation of Deferred@2025091037 after retry RPC after error@1123109815) -> (continuation of Deferred@759490106 after retry RPC after error@52848534) -> (continuation of Deferred@1488403112 after retry RPC after error@282296058) -> (continuation of Deferred@439107564 after retry RPC after error@713482728) -> (continuation of Deferred@1267072106 after retry RPC after error@1076526926) -> (continuation of Deferred@990047224 after retry RPC after error@647414993) -> (continuation of Deferred@1116998341 after retry RPC after error@1373101526) -> (continuation of Deferred@2108076351 after retry RPC after error@1837519382) -> (continuation of Deferred@262586177 after retry RPC after error@1626405390) -> (continuation of Deferred@672628838 after retry RPC after error@2002678160) -> (continuation of Deferred@1303942285 after retry RPC after error@721364926) -> (continuation of Deferred@1048548919 after retry RPC after error@975811225) -> (continuation of Deferred@859588013 after retry RPC after error@1621968547) -> (continuation of Deferred@581024431 after retry RPC after error@1441495320)))
05:27:18.906 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.19.253:39909(127.1.19.253:39909)] disconnecting while in state TERMINATED
05:27:18.908 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.19.252:34617(127.1.19.252:34617)] disconnecting while in state READY
05:27:18.908 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.19.254:45545(127.1.19.254:45545)] disconnecting while in state TERMINATED
05:27:18.912 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
05:27:18.914 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.1.19.252:34617(127.1.19.252:34617)] handling channelInactive
05:27:18.914 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.252:34617(127.1.19.252:34617)] cleaning up while in state READY due to: connection closed
05:27:18.916 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
05:27:18.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:27:18.919440 1103 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task_eGFIx/build/tsan/bin/kudu with pid 1464
05:27:20.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T05:27:20Z chronyd exiting
05:27:21.186 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
05:27:21.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:27:21.188407 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task_eGFIx/build/tsan/bin/kudu with pid 654
05:27:21.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:27:21.216589 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task_eGFIx/build/tsan/bin/kudu with pid 821
05:27:21.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:27:21.246918 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task_eGFIx/build/tsan/bin/kudu with pid 962
05:27:21.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:27:21.276762 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task_eGFIx/build/tsan/bin/kudu with pid 386
05:27:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:27:21.318418 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task_eGFIx/build/tsan/bin/kudu with pid 456
05:27:21.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:27:21.349341 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task_eGFIx/build/tsan/bin/kudu with pid 531
05:27:22.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T05:27:22Z chronyd exiting
05:27:23.582 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=28, TimeoutTracker(timeout=30000, elapsed=29787), Traces: [0ms] refreshing cache from master, [76ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [168ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [177ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [240ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [243ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [597ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [601ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [616ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [622ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [625ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [626ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [628ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [629ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [636ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [641ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [641ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [643ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [644ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [661ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [662ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [664ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [665ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [676ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [681ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [682ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [683ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [684ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [716ms] refreshing cache from master, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [720ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [724ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [726ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [726ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [755ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [761ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [761ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [763ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [764ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [896ms] refreshing cache from master, [896ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [901ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [901ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [903ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [904ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [1056ms] refreshing cache from master, [1057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [1058ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [1059ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [1062ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [1062ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [1064ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [1066ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [1195ms] refreshing cache from master, [1196ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [1196ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [1198ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [1200ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [1201ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [1203ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [1204ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [1616ms] refreshing cache from master, [1616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [1617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [1618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [1620ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [1621ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [1622ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [1623ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [1656ms] refreshing cache from master, [1656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [1658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [1658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [1661ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [1662ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [1663ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [1663ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [2076ms] refreshing cache from master, [2077ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [2077ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [2078ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [2080ms] too many traces: truncated at 100 traces, deferred=Deferred@99617476(state=PENDING, result=null, callback=(continuation of Deferred@79953651 after retry RPC after error@409222695) -> (continuation of Deferred@90765087 after retry RPC after error@13335388) -> (continuation of Deferred@467937744 after retry RPC after error@575191437) -> (continuation of Deferred@1624924884 after retry RPC after error@999351304) -> (continuation of Deferred@16834563 after retry RPC after error@1726199167) -> (continuation of Deferred@1436334005 after retry RPC after error@1771092394) -> (continuation of Deferred@1872120931 after retry RPC after error@346375331) -> (continuation of Deferred@615807133 after retry RPC after error@1208338504) -> (continuation of Deferred@355299455 after retry RPC after error@731098661) -> (continuation of Deferred@744770394 after retry RPC after error@1254461718) -> (continuation of Deferred@873840313 after retry RPC after error@37870744) -> (continuation of Deferred@217147665 after retry RPC after error@1792672952) -> (continuation of Deferred@1940102956 after retry RPC after error@2139764956) -> (continuation of Deferred@2025091037 after retry RPC after error@1123109815) -> (continuation of Deferred@759490106 after retry RPC after error@52848534) -> (continuation of Deferred@1488403112 after retry RPC after error@282296058) -> (continuation of Deferred@439107564 after retry RPC after error@713482728) -> (continuation of Deferred@1267072106 after retry RPC after error@1076526926) -> (continuation of Deferred@990047224 after retry RPC after error@647414993) -> (continuation of Deferred@1116998341 after retry RPC after error@1373101526) -> (continuation of Deferred@2108076351 after retry RPC after error@1837519382) -> (continuation of Deferred@262586177 after retry RPC after error@1626405390) -> (continuation of Deferred@672628838 after retry RPC after error@2002678160) -> (continuation of Deferred@1303942285 after retry RPC after error@721364926) -> (continuation of Deferred@1048548919 after retry RPC after error@975811225) -> (continuation of Deferred@859588013 after retry RPC after error@1621968547) -> (continuation of Deferred@581024431 after retry RPC after error@1441495320), errback=(continuation of Deferred@79953651 after retry RPC after error@409222695) -> (continuation of Deferred@90765087 after retry RPC after error@13335388) -> (continuation of Deferred@467937744 after retry RPC after error@575191437) -> (continuation of Deferred@1624924884 after retry RPC after error@999351304) -> (continuation of Deferred@16834563 after retry RPC after error@1726199167) -> (continuation of Deferred@1436334005 after retry RPC after error@1771092394) -> (continuation of Deferred@1872120931 after retry RPC after error@346375331) -> (continuation of Deferred@615807133 after retry RPC after error@1208338504) -> (continuation of Deferred@355299455 after retry RPC after error@731098661) -> (continuation of Deferred@744770394 after retry RPC after error@1254461718) -> (continuation of Deferred@873840313 after retry RPC after error@37870744) -> (continuation of Deferred@217147665 after retry RPC after error@1792672952) -> (continuation of Deferred@1940102956 after retry RPC after error@2139764956) -> (continuation of Deferred@2025091037 after retry RPC after error@1123109815) -> (continuation of Deferred@759490106 after retry RPC after error@52848534) -> (continuation of Deferred@1488403112 after retry RPC after error@282296058) -> (continuation of Deferred@439107564 after retry RPC after error@713482728) -> (continuation of Deferred@1267072106 after retry RPC after error@1076526926) -> (continuation of Deferred@990047224 after retry RPC after error@647414993) -> (continuation of Deferred@1116998341 after retry RPC after error@1373101526) -> (continuation of Deferred@2108076351 after retry RPC after error@1837519382) -> (continuation of Deferred@262586177 after retry RPC after error@1626405390) -> (continuation of Deferred@672628838 after retry RPC after error@2002678160) -> (continuation of Deferred@1303942285 after retry RPC after error@721364926) -> (continuation of Deferred@1048548919 after retry RPC after error@975811225) -> (continuation of Deferred@859588013 after retry RPC after error@1621968547) -> (continuation of Deferred@581024431 after retry RPC after error@1441495320)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.19.254:45545
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=28, TimeoutTracker(timeout=30000, elapsed=29787), Traces: [0ms] refreshing cache from master, [76ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [168ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [177ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [240ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [243ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [597ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [601ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [616ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [622ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [625ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [626ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [628ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [629ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [636ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [641ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [641ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [643ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [644ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [661ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [662ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [664ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [665ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [676ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [681ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [682ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [683ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [684ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [716ms] refreshing cache from master, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [720ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [724ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [726ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [726ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [755ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [761ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [761ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [763ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [764ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [896ms] refreshing cache from master, [896ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [901ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [901ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [903ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [904ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [1056ms] refreshing cache from master, [1057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [1058ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [1059ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [1062ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [1062ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [1064ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [1066ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [1195ms] refreshing cache from master, [1196ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [1196ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [1198ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [1200ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [1201ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [1203ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [1204ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [1616ms] refreshing cache from master, [1616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [1617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [1618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [1620ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [1621ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [1622ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [1623ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [1656ms] refreshing cache from master, [1656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [1658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [1658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [1661ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [1662ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [1663ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [1663ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [2076ms] refreshing cache from master, [2077ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [2077ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [2078ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [2080ms] too many traces: truncated at 100 traces, deferred=Deferred@99617476(state=PENDING, result=null, callback=(continuation of Deferred@79953651 after retry RPC after error@409222695) -> (continuation of Deferred@90765087 after retry RPC after error@13335388) -> (continuation of Deferred@467937744 after retry RPC after error@575191437) -> (continuation of Deferred@1624924884 after retry RPC after error@999351304) -> (continuation of Deferred@16834563 after retry RPC after error@1726199167) -> (continuation of Deferred@1436334005 after retry RPC after error@1771092394) -> (continuation of Deferred@1872120931 after retry RPC after error@346375331) -> (continuation of Deferred@615807133 after retry RPC after error@1208338504) -> (continuation of Deferred@355299455 after retry RPC after error@731098661) -> (continuation of Deferred@744770394 after retry RPC after error@1254461718) -> (continuation of Deferred@873840313 after retry RPC after error@37870744) -> (continuation of Deferred@217147665 after retry RPC after error@1792672952) -> (continuation of Deferred@1940102956 after retry RPC after error@2139764956) -> (continuation of Deferred@2025091037 after retry RPC after error@1123109815) -> (continuation of Deferred@759490106 after retry RPC after error@52848534) -> (continuation of Deferred@1488403112 after retry RPC after error@282296058) -> (continuation of Deferred@439107564 after retry RPC after error@713482728) -> (continuation of Deferred@1267072106 after retry RPC after error@1076526926) -> (continuation of Deferred@990047224 after retry RPC after error@647414993) -> (continuation of Deferred@1116998341 after retry RPC after error@1373101526) -> (continuation of Deferred@2108076351 after retry RPC after error@1837519382) -> (continuation of Deferred@262586177 after retry RPC after error@1626405390) -> (continuation of Deferred@672628838 after retry RPC after error@2002678160) -> (continuation of Deferred@1303942285 after retry RPC after error@721364926) -> (continuation of Deferred@1048548919 after retry RPC after error@975811225) -> (continuation of Deferred@859588013 after retry RPC after error@1621968547) -> (continuation of Deferred@581024431 after retry RPC after error@1441495320), errback=(continuation of Deferred@79953651 after retry RPC after error@409222695) -> (continuation of Deferred@90765087 after retry RPC after error@13335388) -> (continuation of Deferred@467937744 after retry RPC after error@575191437) -> (continuation of Deferred@1624924884 after retry RPC after error@999351304) -> (continuation of Deferred@16834563 after retry RPC after error@1726199167) -> (continuation of Deferred@1436334005 after retry RPC after error@1771092394) -> (continuation of Deferred@1872120931 after retry RPC after error@346375331) -> (continuation of Deferred@615807133 after retry RPC after error@1208338504) -> (continuation of Deferred@355299455 after retry RPC after error@731098661) -> (continuation of Deferred@744770394 after retry RPC after error@1254461718) -> (continuation of Deferred@873840313 after retry RPC after error@37870744) -> (continuation of Deferred@217147665 after retry RPC after error@1792672952) -> (continuation of Deferred@1940102956 after retry RPC after error@2139764956) -> (continuation of Deferred@2025091037 after retry RPC after error@1123109815) -> (continuation of Deferred@759490106 after retry RPC after error@52848534) -> (continuation of Deferred@1488403112 after retry RPC after error@282296058) -> (continuation of Deferred@439107564 after retry RPC after error@713482728) -> (continuation of Deferred@1267072106 after retry RPC after error@1076526926) -> (continuation of Deferred@990047224 after retry RPC after error@647414993) -> (continuation of Deferred@1116998341 after retry RPC after error@1373101526) -> (continuation of Deferred@2108076351 after retry RPC after error@1837519382) -> (continuation of Deferred@262586177 after retry RPC after error@1626405390) -> (continuation of Deferred@672628838 after retry RPC after error@2002678160) -> (continuation of Deferred@1303942285 after retry RPC after error@721364926) -> (continuation of Deferred@1048548919 after retry RPC after error@975811225) -> (continuation of Deferred@859588013 after retry RPC after error@1621968547) -> (continuation of Deferred@581024431 after retry RPC after error@1441495320)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.19.254:45545
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
05:27:25.006 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
05:27:25.007 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=28, TimeoutTracker(timeout=30000, elapsed=29787), Traces: [0ms] refreshing cache from master, [76ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [168ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [177ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [240ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [243ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [597ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [601ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [616ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [622ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [625ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [626ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [628ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [629ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [636ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [641ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [641ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [643ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [644ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [661ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [662ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [664ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [665ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [676ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [681ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [682ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [683ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [684ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [716ms] refreshing cache from master, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [720ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [724ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [726ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [726ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [755ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [761ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [761ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [763ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [764ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [896ms] refreshing cache from master, [896ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [901ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [901ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [903ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [904ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [1056ms] refreshing cache from master, [1057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [1058ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [1059ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [1062ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [1062ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [1064ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [1066ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [1195ms] refreshing cache from master, [1196ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [1196ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [1198ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [1200ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [1201ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [1203ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [1204ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [1616ms] refreshing cache from master, [1616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [1617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [1618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [1620ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [1621ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [1622ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [1623ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [1656ms] refreshing cache from master, [1656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [1658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [1658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [1661ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [1662ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [1663ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [1663ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [2076ms] refreshing cache from master, [2077ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [2077ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [2078ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [2080ms] too many traces: truncated at 100 traces, deferred=Deferred@99617476(state=PENDING, result=null, callback=(continuation of Deferred@79953651 after retry RPC after error@409222695) -> (continuation of Deferred@90765087 after retry RPC after error@13335388) -> (continuation of Deferred@467937744 after retry RPC after error@575191437) -> (continuation of Deferred@1624924884 after retry RPC after error@999351304) -> (continuation of Deferred@16834563 after retry RPC after error@1726199167) -> (continuation of Deferred@1436334005 after retry RPC after error@1771092394) -> (continuation of Deferred@1872120931 after retry RPC after error@346375331) -> (continuation of Deferred@615807133 after retry RPC after error@1208338504) -> (continuation of Deferred@355299455 after retry RPC after error@731098661) -> (continuation of Deferred@744770394 after retry RPC after error@1254461718) -> (continuation of Deferred@873840313 after retry RPC after error@37870744) -> (continuation of Deferred@217147665 after retry RPC after error@1792672952) -> (continuation of Deferred@1940102956 after retry RPC after error@2139764956) -> (continuation of Deferred@2025091037 after retry RPC after error@1123109815) -> (continuation of Deferred@759490106 after retry RPC after error@52848534) -> (continuation of Deferred@1488403112 after retry RPC after error@282296058) -> (continuation of Deferred@439107564 after retry RPC after error@713482728) -> (continuation of Deferred@1267072106 after retry RPC after error@1076526926) -> (continuation of Deferred@990047224 after retry RPC after error@647414993) -> (continuation of Deferred@1116998341 after retry RPC after error@1373101526) -> (continuation of Deferred@2108076351 after retry RPC after error@1837519382) -> (continuation of Deferred@262586177 after retry RPC after error@1626405390) -> (continuation of Deferred@672628838 after retry RPC after error@2002678160) -> (continuation of Deferred@1303942285 after retry RPC after error@721364926) -> (continuation of Deferred@1048548919 after retry RPC after error@975811225) -> (continuation of Deferred@859588013 after retry RPC after error@1621968547) -> (continuation of Deferred@581024431 after retry RPC after error@1441495320), errback=(continuation of Deferred@79953651 after retry RPC after error@409222695) -> (continuation of Deferred@90765087 after retry RPC after error@13335388) -> (continuation of Deferred@467937744 after retry RPC after error@575191437) -> (continuation of Deferred@1624924884 after retry RPC after error@999351304) -> (continuation of Deferred@16834563 after retry RPC after error@1726199167) -> (continuation of Deferred@1436334005 after retry RPC after error@1771092394) -> (continuation of Deferred@1872120931 after retry RPC after error@346375331) -> (continuation of Deferred@615807133 after retry RPC after error@1208338504) -> (continuation of Deferred@355299455 after retry RPC after error@731098661) -> (continuation of Deferred@744770394 after retry RPC after error@1254461718) -> (continuation of Deferred@873840313 after retry RPC after error@37870744) -> (continuation of Deferred@217147665 after retry RPC after error@1792672952) -> (continuation of Deferred@1940102956 after retry RPC after error@2139764956) -> (continuation of Deferred@2025091037 after retry RPC after error@1123109815) -> (continuation of Deferred@759490106 after retry RPC after error@52848534) -> (continuation of Deferred@1488403112 after retry RPC after error@282296058) -> (continuation of Deferred@439107564 after retry RPC after error@713482728) -> (continuation of Deferred@1267072106 after retry RPC after error@1076526926) -> (continuation of Deferred@990047224 after retry RPC after error@647414993) -> (continuation of Deferred@1116998341 after retry RPC after error@1373101526) -> (continuation of Deferred@2108076351 after retry RPC after error@1837519382) -> (continuation of Deferred@262586177 after retry RPC after error@1626405390) -> (continuation of Deferred@672628838 after retry RPC after error@2002678160) -> (continuation of Deferred@1303942285 after retry RPC after error@721364926) -> (continuation of Deferred@1048548919 after retry RPC after error@975811225) -> (continuation of Deferred@859588013 after retry RPC after error@1621968547) -> (continuation of Deferred@581024431 after retry RPC after error@1441495320)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.19.254:45545
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=28, TimeoutTracker(timeout=30000, elapsed=29787), Traces: [0ms] refreshing cache from master, [76ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [168ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [177ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [240ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [243ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [597ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [601ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [616ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [622ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [625ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [626ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [628ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [629ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [636ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [641ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [641ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [643ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [644ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [661ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [662ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [664ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [665ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [676ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [681ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [682ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [683ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [684ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [716ms] refreshing cache from master, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [720ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [724ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [726ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [726ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [755ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [761ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [761ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [763ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [764ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [896ms] refreshing cache from master, [896ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [901ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [901ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [903ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [904ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [1056ms] refreshing cache from master, [1057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [1058ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [1059ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [1062ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [1062ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [1064ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [1066ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [1195ms] refreshing cache from master, [1196ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [1196ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [1198ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [1200ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [1201ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [1203ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [1204ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [1616ms] refreshing cache from master, [1616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [1617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [1618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [1620ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [1621ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [1622ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [1623ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [1656ms] refreshing cache from master, [1656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [1658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [1658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [1661ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.253:39909: Network error: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909, [1662ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.252:34617: OK, [1663ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.254:45545: Network error: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [1663ms] delaying RPC due to: Service unavailable: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545, [2076ms] refreshing cache from master, [2077ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.254:45545, [2077ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.252:34617, [2078ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.253:39909, [2080ms] too many traces: truncated at 100 traces, deferred=Deferred@99617476(state=PENDING, result=null, callback=(continuation of Deferred@79953651 after retry RPC after error@409222695) -> (continuation of Deferred@90765087 after retry RPC after error@13335388) -> (continuation of Deferred@467937744 after retry RPC after error@575191437) -> (continuation of Deferred@1624924884 after retry RPC after error@999351304) -> (continuation of Deferred@16834563 after retry RPC after error@1726199167) -> (continuation of Deferred@1436334005 after retry RPC after error@1771092394) -> (continuation of Deferred@1872120931 after retry RPC after error@346375331) -> (continuation of Deferred@615807133 after retry RPC after error@1208338504) -> (continuation of Deferred@355299455 after retry RPC after error@731098661) -> (continuation of Deferred@744770394 after retry RPC after error@1254461718) -> (continuation of Deferred@873840313 after retry RPC after error@37870744) -> (continuation of Deferred@217147665 after retry RPC after error@1792672952) -> (continuation of Deferred@1940102956 after retry RPC after error@2139764956) -> (continuation of Deferred@2025091037 after retry RPC after error@1123109815) -> (continuation of Deferred@759490106 after retry RPC after error@52848534) -> (continuation of Deferred@1488403112 after retry RPC after error@282296058) -> (continuation of Deferred@439107564 after retry RPC after error@713482728) -> (continuation of Deferred@1267072106 after retry RPC after error@1076526926) -> (continuation of Deferred@990047224 after retry RPC after error@647414993) -> (continuation of Deferred@1116998341 after retry RPC after error@1373101526) -> (continuation of Deferred@2108076351 after retry RPC after error@1837519382) -> (continuation of Deferred@262586177 after retry RPC after error@1626405390) -> (continuation of Deferred@672628838 after retry RPC after error@2002678160) -> (continuation of Deferred@1303942285 after retry RPC after error@721364926) -> (continuation of Deferred@1048548919 after retry RPC after error@975811225) -> (continuation of Deferred@859588013 after retry RPC after error@1621968547) -> (continuation of Deferred@581024431 after retry RPC after error@1441495320), errback=(continuation of Deferred@79953651 after retry RPC after error@409222695) -> (continuation of Deferred@90765087 after retry RPC after error@13335388) -> (continuation of Deferred@467937744 after retry RPC after error@575191437) -> (continuation of Deferred@1624924884 after retry RPC after error@999351304) -> (continuation of Deferred@16834563 after retry RPC after error@1726199167) -> (continuation of Deferred@1436334005 after retry RPC after error@1771092394) -> (continuation of Deferred@1872120931 after retry RPC after error@346375331) -> (continuation of Deferred@615807133 after retry RPC after error@1208338504) -> (continuation of Deferred@355299455 after retry RPC after error@731098661) -> (continuation of Deferred@744770394 after retry RPC after error@1254461718) -> (continuation of Deferred@873840313 after retry RPC after error@37870744) -> (continuation of Deferred@217147665 after retry RPC after error@1792672952) -> (continuation of Deferred@1940102956 after retry RPC after error@2139764956) -> (continuation of Deferred@2025091037 after retry RPC after error@1123109815) -> (continuation of Deferred@759490106 after retry RPC after error@52848534) -> (continuation of Deferred@1488403112 after retry RPC after error@282296058) -> (continuation of Deferred@439107564 after retry RPC after error@713482728) -> (continuation of Deferred@1267072106 after retry RPC after error@1076526926) -> (continuation of Deferred@990047224 after retry RPC after error@647414993) -> (continuation of Deferred@1116998341 after retry RPC after error@1373101526) -> (continuation of Deferred@2108076351 after retry RPC after error@1837519382) -> (continuation of Deferred@262586177 after retry RPC after error@1626405390) -> (continuation of Deferred@672628838 after retry RPC after error@2002678160) -> (continuation of Deferred@1303942285 after retry RPC after error@721364926) -> (continuation of Deferred@1048548919 after retry RPC after error@975811225) -> (continuation of Deferred@859588013 after retry RPC after error@1621968547) -> (continuation of Deferred@581024431 after retry RPC after error@1441495320)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.19.254:45545,127.1.19.252:34617,127.1.19.253:39909) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.253:39909(127.1.19.253:39909): Connection refused: /127.1.19.253:39909,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.254:45545(127.1.19.254:45545): Connection refused: /127.1.19.254:45545
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.19.254:45545
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)